tls/tls_sw.c

   1 /*
   2  * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
   3  * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
   4  * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
   5  * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
   6  * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
   7  * Copyright (c) 2018, Covalent IO, Inc. http://covalent.io
   8  *
   9  * This software is available to you under a choice of one of two
  10  * licenses.  You may choose to be licensed under the terms of the GNU
  11  * General Public License (GPL) Version 2, available from the file
  12  * COPYING in the main directory of this source tree, or the
  13  * OpenIB.org BSD license below:
  14  *
  15  *     Redistribution and use in source and binary forms, with or
  16  *     without modification, are permitted provided that the following
  17  *     conditions are met:
  18  *
  19  *      - Redistributions of source code must retain the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer.
  22  *
  23  *      - Redistributions in binary form must reproduce the above
  24  *        copyright notice, this list of conditions and the following
  25  *        disclaimer in the documentation and/or other materials
  26  *        provided with the distribution.
  27  *
  28  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  29  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  30  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  31  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  32  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  33  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  34  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  35  * SOFTWARE.
  36  */
  37
  38 #include <linux/bug.h>
  39 #include <linux/sched/signal.h>
  40 #include <linux/module.h>
  41 #include <linux/kernel.h>
  42 #include <linux/splice.h>
  43 #include <crypto/aead.h>
  44
  45 #include <net/strparser.h>
  46 #include <net/tls.h>
  47
  48 #include "tls.h"
  49
  50 struct tls_decrypt_arg {
  51         struct_group(inargs,
  52         bool zc;
  53         bool async;
  54         bool async_done;
  55         u8 tail;
  56         );
  57
  58         struct sk_buff *skb;
  59 };
  60
  61 struct tls_decrypt_ctx {
  62         struct sock *sk;
  63         u8 iv[MAX_IV_SIZE];
  64         u8 aad[TLS_MAX_AAD_SIZE];
  65         u8 tail;
  66         bool free_sgout;
  67         struct scatterlist sg[];
  68 };
  69
  70 noinline void tls_err_abort(struct sock *sk, int err)
  71 {
  72         WARN_ON_ONCE(err >= 0);
  73         /* sk->sk_err should contain a positive error code. */
  74         WRITE_ONCE(sk->sk_err, -err);
  75         /* Paired with smp_rmb() in tcp_poll() */
  76         smp_wmb();
  77         sk_error_report(sk);
  78 }
  79
  80 static int __skb_nsg(struct sk_buff *skb, int offset, int len,
  81                      unsigned int recursion_level)
  82 {
  83         int start = skb_headlen(skb);
  84         int i, chunk = start - offset;
  85         struct sk_buff *frag_iter;
  86         int elt = 0;
  87
  88         if (unlikely(recursion_level >= 24))
  89                 return -EMSGSIZE;
  90
  91         if (chunk > 0) {
  92                 if (chunk > len)
  93                         chunk = len;
  94                 elt++;
  95                 len -= chunk;
  96                 if (len == 0)
  97                         return elt;
  98                 offset += chunk;
  99         }
 100
 101         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 102                 int end;
 103
 104                 WARN_ON(start > offset + len);
 105
 106                 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
 107                 chunk = end - offset;
 108                 if (chunk > 0) {
 109                         if (chunk > len)
 110                                 chunk = len;
 111                         elt++;
 112                         len -= chunk;
 113                         if (len == 0)
 114                                 return elt;
 115                         offset += chunk;
 116                 }
 117                 start = end;
 118         }
 119
 120         if (unlikely(skb_has_frag_list(skb))) {
 121                 skb_walk_frags(skb, frag_iter) {
 122                         int end, ret;
 123
 124                         WARN_ON(start > offset + len);
 125
 126                         end = start + frag_iter->len;
 127                         chunk = end - offset;
 128                         if (chunk > 0) {
 129                                 if (chunk > len)
 130                                         chunk = len;
 131                                 ret = __skb_nsg(frag_iter, offset - start, chunk,
 132                                                 recursion_level + 1);
 133                                 if (unlikely(ret < 0))
 134                                         return ret;
 135                                 elt += ret;
 136                                 len -= chunk;
 137                                 if (len == 0)
 138                                         return elt;
 139                                 offset += chunk;
 140                         }
 141                         start = end;
 142                 }
 143         }
 144         BUG_ON(len);
 145         return elt;
 146 }
 147
 148 /* Return the number of scatterlist elements required to completely map the
 149  * skb, or -EMSGSIZE if the recursion depth is exceeded.
 150  */
 151 static int skb_nsg(struct sk_buff *skb, int offset, int len)
 152 {
 153         return __skb_nsg(skb, offset, len, 0);
 154 }
 155
 156 static int tls_padding_length(struct tls_prot_info *prot, struct sk_buff *skb,
 157                               struct tls_decrypt_arg *darg)
 158 {
 159         struct strp_msg *rxm = strp_msg(skb);
 160         struct tls_msg *tlm = tls_msg(skb);
 161         int sub = 0;
 162
 163         /* Determine zero-padding length */
 164         if (prot->version == TLS_1_3_VERSION) {
 165                 int offset = rxm->full_len - TLS_TAG_SIZE - 1;
 166                 char content_type = darg->zc ? darg->tail : 0;
 167                 int err;
 168
 169                 while (content_type == 0) {
 170                         if (offset < prot->prepend_size)
 171                                 return -EBADMSG;
 172                         err = skb_copy_bits(skb, rxm->offset + offset,
 173                                             &content_type, 1);
 174                         if (err)
 175                                 return err;
 176                         if (content_type)
 177                                 break;
 178                         sub++;
 179                         offset--;
 180                 }
 181                 tlm->control = content_type;
 182         }
 183         return sub;
 184 }
 185
 186 static void tls_decrypt_done(crypto_completion_data_t *data, int err)
 187 {
 188         struct aead_request *aead_req = crypto_get_completion_data(data);
 189         struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
 190         struct scatterlist *sgout = aead_req->dst;
 191         struct tls_sw_context_rx *ctx;
 192         struct tls_decrypt_ctx *dctx;
 193         struct tls_context *tls_ctx;
 194         struct scatterlist *sg;
 195         unsigned int pages;
 196         struct sock *sk;
 197         int aead_size;
 198
 199         /* If requests get too backlogged crypto API returns -EBUSY and calls
 200          * ->complete(-EINPROGRESS) immediately followed by ->complete(0)
 201          * to make waiting for backlog to flush with crypto_wait_req() easier.
 202          * First wait converts -EBUSY -> -EINPROGRESS, and the second one
 203          * -EINPROGRESS -> 0.
 204          * We have a single struct crypto_async_request per direction, this
 205          * scheme doesn't help us, so just ignore the first ->complete().
 206          */
 207         if (err == -EINPROGRESS)
 208                 return;
 209
 210         aead_size = sizeof(*aead_req) + crypto_aead_reqsize(aead);
 211         aead_size = ALIGN(aead_size, __alignof__(*dctx));
 212         dctx = (void *)((u8 *)aead_req + aead_size);
 213
 214         sk = dctx->sk;
 215         tls_ctx = tls_get_ctx(sk);
 216         ctx = tls_sw_ctx_rx(tls_ctx);
 217
 218         /* Propagate if there was an err */
 219         if (err) {
 220                 if (err == -EBADMSG)
 221                         TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTERROR);
 222                 ctx->async_wait.err = err;
 223                 tls_err_abort(sk, err);
 224         }
 225
 226         /* Free the destination pages if skb was not decrypted inplace */
 227         if (dctx->free_sgout) {
 228                 /* Skip the first S/G entry as it points to AAD */
 229                 for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) {
 230                         if (!sg)
 231                                 break;
 232                         put_page(sg_page(sg));
 233                 }
 234         }
 235
 236         kfree(aead_req);
 237
 238         if (atomic_dec_and_test(&ctx->decrypt_pending))
 239                 complete(&ctx->async_wait.completion);
 240 }
 241
 242 static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx)
 243 {
 244         if (!atomic_dec_and_test(&ctx->decrypt_pending))
 245                 crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
 246         atomic_inc(&ctx->decrypt_pending);
 247
 248         return ctx->async_wait.err;
 249 }
 250
 251 static int tls_do_decryption(struct sock *sk,
 252                              struct scatterlist *sgin,
 253                              struct scatterlist *sgout,
 254                              char *iv_recv,
 255                              size_t data_len,
 256                              struct aead_request *aead_req,
 257                              struct tls_decrypt_arg *darg)
 258 {
 259         struct tls_context *tls_ctx = tls_get_ctx(sk);
 260         struct tls_prot_info *prot = &tls_ctx->prot_info;
 261         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
 262         int ret;
 263
 264         aead_request_set_tfm(aead_req, ctx->aead_recv);
 265         aead_request_set_ad(aead_req, prot->aad_size);
 266         aead_request_set_crypt(aead_req, sgin, sgout,
 267                                data_len + prot->tag_size,
 268                                (u8 *)iv_recv);
 269
 270         if (darg->async) {
 271                 aead_request_set_callback(aead_req,
 272                                           CRYPTO_TFM_REQ_MAY_BACKLOG,
 273                                           tls_decrypt_done, aead_req);
 274                 DEBUG_NET_WARN_ON_ONCE(atomic_read(&ctx->decrypt_pending) < 1);
 275                 atomic_inc(&ctx->decrypt_pending);
 276         } else {
 277                 aead_request_set_callback(aead_req,
 278                                           CRYPTO_TFM_REQ_MAY_BACKLOG,
 279                                           crypto_req_done, &ctx->async_wait);
 280         }
 281
 282         ret = crypto_aead_decrypt(aead_req);
 283         if (ret == -EINPROGRESS)
 284                 return 0;
 285
 286         if (ret == -EBUSY) {
 287                 ret = tls_decrypt_async_wait(ctx);
 288                 darg->async_done = true;
 289                 /* all completions have run, we're not doing async anymore */
 290                 darg->async = false;
 291                 return ret;
 292                 ret = ret ?: -EINPROGRESS;
 293         }
 294
 295         atomic_dec(&ctx->decrypt_pending);
 296         darg->async = false;
 297
 298         return ret;
 299 }
 300
 301 static void tls_trim_both_msgs(struct sock *sk, int target_size)
 302 {
 303         struct tls_context *tls_ctx = tls_get_ctx(sk);
 304         struct tls_prot_info *prot = &tls_ctx->prot_info;
 305         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 306         struct tls_rec *rec = ctx->open_rec;
 307
 308         sk_msg_trim(sk, &rec->msg_plaintext, target_size);
 309         if (target_size > 0)
 310                 target_size += prot->overhead_size;
 311         sk_msg_trim(sk, &rec->msg_encrypted, target_size);
 312 }
 313
 314 static int tls_alloc_encrypted_msg(struct sock *sk, int len)
 315 {
 316         struct tls_context *tls_ctx = tls_get_ctx(sk);
 317         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 318         struct tls_rec *rec = ctx->open_rec;
 319         struct sk_msg *msg_en = &rec->msg_encrypted;
 320
 321         return sk_msg_alloc(sk, msg_en, len, 0);
 322 }
 323
 324 static int tls_clone_plaintext_msg(struct sock *sk, int required)
 325 {
 326         struct tls_context *tls_ctx = tls_get_ctx(sk);
 327         struct tls_prot_info *prot = &tls_ctx->prot_info;
 328         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 329         struct tls_rec *rec = ctx->open_rec;
 330         struct sk_msg *msg_pl = &rec->msg_plaintext;
 331         struct sk_msg *msg_en = &rec->msg_encrypted;
 332         int skip, len;
 333
 334         /* We add page references worth len bytes from encrypted sg
 335          * at the end of plaintext sg. It is guaranteed that msg_en
 336          * has enough required room (ensured by caller).
 337          */
 338         len = required - msg_pl->sg.size;
 339
 340         /* Skip initial bytes in msg_en's data to be able to use
 341          * same offset of both plain and encrypted data.
 342          */
 343         skip = prot->prepend_size + msg_pl->sg.size;
 344
 345         return sk_msg_clone(sk, msg_pl, msg_en, skip, len);
 346 }
 347
 348 static struct tls_rec *tls_get_rec(struct sock *sk)
 349 {
 350         struct tls_context *tls_ctx = tls_get_ctx(sk);
 351         struct tls_prot_info *prot = &tls_ctx->prot_info;
 352         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 353         struct sk_msg *msg_pl, *msg_en;
 354         struct tls_rec *rec;
 355         int mem_size;
 356
 357         mem_size = sizeof(struct tls_rec) + crypto_aead_reqsize(ctx->aead_send);
 358
 359         rec = kzalloc(mem_size, sk->sk_allocation);
 360         if (!rec)
 361                 return NULL;
 362
 363         msg_pl = &rec->msg_plaintext;
 364         msg_en = &rec->msg_encrypted;
 365
 366         sk_msg_init(msg_pl);
 367         sk_msg_init(msg_en);
 368
 369         sg_init_table(rec->sg_aead_in, 2);
 370         sg_set_buf(&rec->sg_aead_in[0], rec->aad_space, prot->aad_size);
 371         sg_unmark_end(&rec->sg_aead_in[1]);
 372
 373         sg_init_table(rec->sg_aead_out, 2);
 374         sg_set_buf(&rec->sg_aead_out[0], rec->aad_space, prot->aad_size);
 375         sg_unmark_end(&rec->sg_aead_out[1]);
 376
 377         rec->sk = sk;
 378
 379         return rec;
 380 }
 381
 382 static void tls_free_rec(struct sock *sk, struct tls_rec *rec)
 383 {
 384         sk_msg_free(sk, &rec->msg_encrypted);
 385         sk_msg_free(sk, &rec->msg_plaintext);
 386         kfree(rec);
 387 }
 388
 389 static void tls_free_open_rec(struct sock *sk)
 390 {
 391         struct tls_context *tls_ctx = tls_get_ctx(sk);
 392         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 393         struct tls_rec *rec = ctx->open_rec;
 394
 395         if (rec) {
 396                 tls_free_rec(sk, rec);
 397                 ctx->open_rec = NULL;
 398         }
 399 }
 400
 401 int tls_tx_records(struct sock *sk, int flags)
 402 {
 403         struct tls_context *tls_ctx = tls_get_ctx(sk);
 404         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 405         struct tls_rec *rec, *tmp;
 406         struct sk_msg *msg_en;
 407         int tx_flags, rc = 0;
 408
 409         if (tls_is_partially_sent_record(tls_ctx)) {
 410                 rec = list_first_entry(&ctx->tx_list,
 411                                        struct tls_rec, list);
 412
 413                 if (flags == -1)
 414                         tx_flags = rec->tx_flags;
 415                 else
 416                         tx_flags = flags;
 417
 418                 rc = tls_push_partial_record(sk, tls_ctx, tx_flags);
 419                 if (rc)
 420                         goto tx_err;
 421
 422                 /* Full record has been transmitted.
 423                  * Remove the head of tx_list
 424                  */
 425                 list_del(&rec->list);
 426                 sk_msg_free(sk, &rec->msg_plaintext);
 427                 kfree(rec);
 428         }
 429
 430         /* Tx all ready records */
 431         list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) {
 432                 if (READ_ONCE(rec->tx_ready)) {
 433                         if (flags == -1)
 434                                 tx_flags = rec->tx_flags;
 435                         else
 436                                 tx_flags = flags;
 437
 438                         msg_en = &rec->msg_encrypted;
 439                         rc = tls_push_sg(sk, tls_ctx,
 440                                          &msg_en->sg.data[msg_en->sg.curr],
 441                                          0, tx_flags);
 442                         if (rc)
 443                                 goto tx_err;
 444
 445                         list_del(&rec->list);
 446                         sk_msg_free(sk, &rec->msg_plaintext);
 447                         kfree(rec);
 448                 } else {
 449                         break;
 450                 }
 451         }
 452
 453 tx_err:
 454         if (rc < 0 && rc != -EAGAIN)
 455                 tls_err_abort(sk, -EBADMSG);
 456
 457         return rc;
 458 }
 459
 460 static void tls_encrypt_done(crypto_completion_data_t *data, int err)
 461 {
 462         struct aead_request *aead_req = crypto_get_completion_data(data);
 463         struct tls_sw_context_tx *ctx;
 464         struct tls_context *tls_ctx;
 465         struct tls_prot_info *prot;
 466         struct scatterlist *sge;
 467         struct sk_msg *msg_en;
 468         struct tls_rec *rec;
 469         struct sock *sk;
 470
 471         if (err == -EINPROGRESS) /* see the comment in tls_decrypt_done() */
 472                 return;
 473
 474         rec = container_of(aead_req, struct tls_rec, aead_req);
 475         msg_en = &rec->msg_encrypted;
 476
 477         sk = rec->sk;
 478         tls_ctx = tls_get_ctx(sk);
 479         prot = &tls_ctx->prot_info;
 480         ctx = tls_sw_ctx_tx(tls_ctx);
 481
 482         sge = sk_msg_elem(msg_en, msg_en->sg.curr);
 483         sge->offset -= prot->prepend_size;
 484         sge->length += prot->prepend_size;
 485
 486         /* Check if error is previously set on socket */
 487         if (err || sk->sk_err) {
 488                 rec = NULL;
 489
 490                 /* If err is already set on socket, return the same code */
 491                 if (sk->sk_err) {
 492                         ctx->async_wait.err = -sk->sk_err;
 493                 } else {
 494                         ctx->async_wait.err = err;
 495                         tls_err_abort(sk, err);
 496                 }
 497         }
 498
 499         if (rec) {
 500                 struct tls_rec *first_rec;
 501
 502                 /* Mark the record as ready for transmission */
 503                 smp_store_mb(rec->tx_ready, true);
 504
 505                 /* If received record is at head of tx_list, schedule tx */
 506                 first_rec = list_first_entry(&ctx->tx_list,
 507                                              struct tls_rec, list);
 508                 if (rec == first_rec) {
 509                         /* Schedule the transmission */
 510                         if (!test_and_set_bit(BIT_TX_SCHEDULED,
 511                                               &ctx->tx_bitmask))
 512                                 schedule_delayed_work(&ctx->tx_work.work, 1);
 513                 }
 514         }
 515
 516         if (atomic_dec_and_test(&ctx->encrypt_pending))
 517                 complete(&ctx->async_wait.completion);
 518 }
 519
 520 static int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx)
 521 {
 522         if (!atomic_dec_and_test(&ctx->encrypt_pending))
 523                 crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
 524         atomic_inc(&ctx->encrypt_pending);
 525
 526         return ctx->async_wait.err;
 527 }
 528
 529 static int tls_do_encryption(struct sock *sk,
 530                              struct tls_context *tls_ctx,
 531                              struct tls_sw_context_tx *ctx,
 532                              struct aead_request *aead_req,
 533                              size_t data_len, u32 start)
 534 {
 535         struct tls_prot_info *prot = &tls_ctx->prot_info;
 536         struct tls_rec *rec = ctx->open_rec;
 537         struct sk_msg *msg_en = &rec->msg_encrypted;
 538         struct scatterlist *sge = sk_msg_elem(msg_en, start);
 539         int rc, iv_offset = 0;
 540
 541         /* For CCM based ciphers, first byte of IV is a constant */
 542         switch (prot->cipher_type) {
 543         case TLS_CIPHER_AES_CCM_128:
 544                 rec->iv_data[0] = TLS_AES_CCM_IV_B0_BYTE;
 545                 iv_offset = 1;
 546                 break;
 547         case TLS_CIPHER_SM4_CCM:
 548                 rec->iv_data[0] = TLS_SM4_CCM_IV_B0_BYTE;
 549                 iv_offset = 1;
 550                 break;
 551         }
 552
 553         memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv,
 554                prot->iv_size + prot->salt_size);
 555
 556         tls_xor_iv_with_seq(prot, rec->iv_data + iv_offset,
 557                             tls_ctx->tx.rec_seq);
 558
 559         sge->offset += prot->prepend_size;
 560         sge->length -= prot->prepend_size;
 561
 562         msg_en->sg.curr = start;
 563
 564         aead_request_set_tfm(aead_req, ctx->aead_send);
 565         aead_request_set_ad(aead_req, prot->aad_size);
 566         aead_request_set_crypt(aead_req, rec->sg_aead_in,
 567                                rec->sg_aead_out,
 568                                data_len, rec->iv_data);
 569
 570         aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 571                                   tls_encrypt_done, aead_req);
 572
 573         /* Add the record in tx_list */
 574         list_add_tail((struct list_head *)&rec->list, &ctx->tx_list);
 575         DEBUG_NET_WARN_ON_ONCE(atomic_read(&ctx->encrypt_pending) < 1);
 576         atomic_inc(&ctx->encrypt_pending);
 577
 578         rc = crypto_aead_encrypt(aead_req);
 579         if (rc == -EBUSY) {
 580                 rc = tls_encrypt_async_wait(ctx);
 581                 rc = rc ?: -EINPROGRESS;
 582         }
 583         if (!rc || rc != -EINPROGRESS) {
 584                 atomic_dec(&ctx->encrypt_pending);
 585                 sge->offset -= prot->prepend_size;
 586                 sge->length += prot->prepend_size;
 587         }
 588
 589         if (!rc) {
 590                 WRITE_ONCE(rec->tx_ready, true);
 591         } else if (rc != -EINPROGRESS) {
 592                 list_del(&rec->list);
 593                 return rc;
 594         }
 595
 596         /* Unhook the record from context if encryption is not failure */
 597         ctx->open_rec = NULL;
 598         tls_advance_record_sn(sk, prot, &tls_ctx->tx);
 599         return rc;
 600 }
 601
 602 static int tls_split_open_record(struct sock *sk, struct tls_rec *from,
 603                                  struct tls_rec **to, struct sk_msg *msg_opl,
 604                                  struct sk_msg *msg_oen, u32 split_point,
 605                                  u32 tx_overhead_size, u32 *orig_end)
 606 {
 607         u32 i, j, bytes = 0, apply = msg_opl->apply_bytes;
 608         struct scatterlist *sge, *osge, *nsge;
 609         u32 orig_size = msg_opl->sg.size;
 610         struct scatterlist tmp = { };
 611         struct sk_msg *msg_npl;
 612         struct tls_rec *new;
 613         int ret;
 614
 615         new = tls_get_rec(sk);
 616         if (!new)
 617                 return -ENOMEM;
 618         ret = sk_msg_alloc(sk, &new->msg_encrypted, msg_opl->sg.size +
 619                            tx_overhead_size, 0);
 620         if (ret < 0) {
 621                 tls_free_rec(sk, new);
 622                 return ret;
 623         }
 624
 625         *orig_end = msg_opl->sg.end;
 626         i = msg_opl->sg.start;
 627         sge = sk_msg_elem(msg_opl, i);
 628         while (apply && sge->length) {
 629                 if (sge->length > apply) {
 630                         u32 len = sge->length - apply;
 631
 632                         get_page(sg_page(sge));
 633                         sg_set_page(&tmp, sg_page(sge), len,
 634                                     sge->offset + apply);
 635                         sge->length = apply;
 636                         bytes += apply;
 637                         apply = 0;
 638                 } else {
 639                         apply -= sge->length;
 640                         bytes += sge->length;
 641                 }
 642
 643                 sk_msg_iter_var_next(i);
 644                 if (i == msg_opl->sg.end)
 645                         break;
 646                 sge = sk_msg_elem(msg_opl, i);
 647         }
 648
 649         msg_opl->sg.end = i;
 650         msg_opl->sg.curr = i;
 651         msg_opl->sg.copybreak = 0;
 652         msg_opl->apply_bytes = 0;
 653         msg_opl->sg.size = bytes;
 654
 655         msg_npl = &new->msg_plaintext;
 656         msg_npl->apply_bytes = apply;
 657         msg_npl->sg.size = orig_size - bytes;
 658
 659         j = msg_npl->sg.start;
 660         nsge = sk_msg_elem(msg_npl, j);
 661         if (tmp.length) {
 662                 memcpy(nsge, &tmp, sizeof(*nsge));
 663                 sk_msg_iter_var_next(j);
 664                 nsge = sk_msg_elem(msg_npl, j);
 665         }
 666
 667         osge = sk_msg_elem(msg_opl, i);
 668         while (osge->length) {
 669                 memcpy(nsge, osge, sizeof(*nsge));
 670                 sg_unmark_end(nsge);
 671                 sk_msg_iter_var_next(i);
 672                 sk_msg_iter_var_next(j);
 673                 if (i == *orig_end)
 674                         break;
 675                 osge = sk_msg_elem(msg_opl, i);
 676                 nsge = sk_msg_elem(msg_npl, j);
 677         }
 678
 679         msg_npl->sg.end = j;
 680         msg_npl->sg.curr = j;
 681         msg_npl->sg.copybreak = 0;
 682
 683         *to = new;
 684         return 0;
 685 }
 686
 687 static void tls_merge_open_record(struct sock *sk, struct tls_rec *to,
 688                                   struct tls_rec *from, u32 orig_end)
 689 {
 690         struct sk_msg *msg_npl = &from->msg_plaintext;
 691         struct sk_msg *msg_opl = &to->msg_plaintext;
 692         struct scatterlist *osge, *nsge;
 693         u32 i, j;
 694
 695         i = msg_opl->sg.end;
 696         sk_msg_iter_var_prev(i);
 697         j = msg_npl->sg.start;
 698
 699         osge = sk_msg_elem(msg_opl, i);
 700         nsge = sk_msg_elem(msg_npl, j);
 701
 702         if (sg_page(osge) == sg_page(nsge) &&
 703             osge->offset + osge->length == nsge->offset) {
 704                 osge->length += nsge->length;
 705                 put_page(sg_page(nsge));
 706         }
 707
 708         msg_opl->sg.end = orig_end;
 709         msg_opl->sg.curr = orig_end;
 710         msg_opl->sg.copybreak = 0;
 711         msg_opl->apply_bytes = msg_opl->sg.size + msg_npl->sg.size;
 712         msg_opl->sg.size += msg_npl->sg.size;
 713
 714         sk_msg_free(sk, &to->msg_encrypted);
 715         sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted);
 716
 717         kfree(from);
 718 }
 719
 720 static int tls_push_record(struct sock *sk, int flags,
 721                            unsigned char record_type)
 722 {
 723         struct tls_context *tls_ctx = tls_get_ctx(sk);
 724         struct tls_prot_info *prot = &tls_ctx->prot_info;
 725         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 726         struct tls_rec *rec = ctx->open_rec, *tmp = NULL;
 727         u32 i, split_point, orig_end;
 728         struct sk_msg *msg_pl, *msg_en;
 729         struct aead_request *req;
 730         bool split;
 731         int rc;
 732
 733         if (!rec)
 734                 return 0;
 735
 736         msg_pl = &rec->msg_plaintext;
 737         msg_en = &rec->msg_encrypted;
 738
 739         split_point = msg_pl->apply_bytes;
 740         split = split_point && split_point < msg_pl->sg.size;
 741         if (unlikely((!split &&
 742                       msg_pl->sg.size +
 743                       prot->overhead_size > msg_en->sg.size) ||
 744                      (split &&
 745                       split_point +
 746                       prot->overhead_size > msg_en->sg.size))) {
 747                 split = true;
 748                 split_point = msg_en->sg.size;
 749         }
 750         if (split) {
 751                 rc = tls_split_open_record(sk, rec, &tmp, msg_pl, msg_en,
 752                                            split_point, prot->overhead_size,
 753                                            &orig_end);
 754                 if (rc < 0)
 755                         return rc;
 756                 /* This can happen if above tls_split_open_record allocates
 757                  * a single large encryption buffer instead of two smaller
 758                  * ones. In this case adjust pointers and continue without
 759                  * split.
 760                  */
 761                 if (!msg_pl->sg.size) {
 762                         tls_merge_open_record(sk, rec, tmp, orig_end);
 763                         msg_pl = &rec->msg_plaintext;
 764                         msg_en = &rec->msg_encrypted;
 765                         split = false;
 766                 }
 767                 sk_msg_trim(sk, msg_en, msg_pl->sg.size +
 768                             prot->overhead_size);
 769         }
 770
 771         rec->tx_flags = flags;
 772         req = &rec->aead_req;
 773
 774         i = msg_pl->sg.end;
 775         sk_msg_iter_var_prev(i);
 776
 777         rec->content_type = record_type;
 778         if (prot->version == TLS_1_3_VERSION) {
 779                 /* Add content type to end of message.  No padding added */
 780                 sg_set_buf(&rec->sg_content_type, &rec->content_type, 1);
 781                 sg_mark_end(&rec->sg_content_type);
 782                 sg_chain(msg_pl->sg.data, msg_pl->sg.end + 1,
 783                          &rec->sg_content_type);
 784         } else {
 785                 sg_mark_end(sk_msg_elem(msg_pl, i));
 786         }
 787
 788         if (msg_pl->sg.end < msg_pl->sg.start) {
 789                 sg_chain(&msg_pl->sg.data[msg_pl->sg.start],
 790                          MAX_SKB_FRAGS - msg_pl->sg.start + 1,
 791                          msg_pl->sg.data);
 792         }
 793
 794         i = msg_pl->sg.start;
 795         sg_chain(rec->sg_aead_in, 2, &msg_pl->sg.data[i]);
 796
 797         i = msg_en->sg.end;
 798         sk_msg_iter_var_prev(i);
 799         sg_mark_end(sk_msg_elem(msg_en, i));
 800
 801         i = msg_en->sg.start;
 802         sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]);
 803
 804         tls_make_aad(rec->aad_space, msg_pl->sg.size + prot->tail_size,
 805                      tls_ctx->tx.rec_seq, record_type, prot);
 806
 807         tls_fill_prepend(tls_ctx,
 808                          page_address(sg_page(&msg_en->sg.data[i])) +
 809                          msg_en->sg.data[i].offset,
 810                          msg_pl->sg.size + prot->tail_size,
 811                          record_type);
 812
 813         tls_ctx->pending_open_record_frags = false;
 814
 815         rc = tls_do_encryption(sk, tls_ctx, ctx, req,
 816                                msg_pl->sg.size + prot->tail_size, i);
 817         if (rc < 0) {
 818                 if (rc != -EINPROGRESS) {
 819                         tls_err_abort(sk, -EBADMSG);
 820                         if (split) {
 821                                 tls_ctx->pending_open_record_frags = true;
 822                                 tls_merge_open_record(sk, rec, tmp, orig_end);
 823                         }
 824                 }
 825                 ctx->async_capable = 1;
 826                 return rc;
 827         } else if (split) {
 828                 msg_pl = &tmp->msg_plaintext;
 829                 msg_en = &tmp->msg_encrypted;
 830                 sk_msg_trim(sk, msg_en, msg_pl->sg.size + prot->overhead_size);
 831                 tls_ctx->pending_open_record_frags = true;
 832                 ctx->open_rec = tmp;
 833         }
 834
 835         return tls_tx_records(sk, flags);
 836 }
 837
 838 static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,
 839                                bool full_record, u8 record_type,
 840                                ssize_t *copied, int flags)
 841 {
 842         struct tls_context *tls_ctx = tls_get_ctx(sk);
 843         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 844         struct sk_msg msg_redir = { };
 845         struct sk_psock *psock;
 846         struct sock *sk_redir;
 847         struct tls_rec *rec;
 848         bool enospc, policy, redir_ingress;
 849         int err = 0, send;
 850         u32 delta = 0;
 851
 852         policy = !(flags & MSG_SENDPAGE_NOPOLICY);
 853         psock = sk_psock_get(sk);
 854         if (!psock || !policy) {
 855                 err = tls_push_record(sk, flags, record_type);
 856                 if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) {
 857                         *copied -= sk_msg_free(sk, msg);
 858                         tls_free_open_rec(sk);
 859                         err = -sk->sk_err;
 860                 }
 861                 if (psock)
 862                         sk_psock_put(sk, psock);
 863                 return err;
 864         }
 865 more_data:
 866         enospc = sk_msg_full(msg);
 867         if (psock->eval == __SK_NONE) {
 868                 delta = msg->sg.size;
 869                 psock->eval = sk_psock_msg_verdict(sk, psock, msg);
 870                 delta -= msg->sg.size;
 871         }
 872         if (msg->cork_bytes && msg->cork_bytes > msg->sg.size &&
 873             !enospc && !full_record) {
 874                 err = -ENOSPC;
 875                 goto out_err;
 876         }
 877         msg->cork_bytes = 0;
 878         send = msg->sg.size;
 879         if (msg->apply_bytes && msg->apply_bytes < send)
 880                 send = msg->apply_bytes;
 881
 882         switch (psock->eval) {
 883         case __SK_PASS:
 884                 err = tls_push_record(sk, flags, record_type);
 885                 if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) {
 886                         *copied -= sk_msg_free(sk, msg);
 887                         tls_free_open_rec(sk);
 888                         err = -sk->sk_err;
 889                         goto out_err;
 890                 }
 891                 break;
 892         case __SK_REDIRECT:
 893                 redir_ingress = psock->redir_ingress;
 894                 sk_redir = psock->sk_redir;
 895                 memcpy(&msg_redir, msg, sizeof(*msg));
 896                 if (msg->apply_bytes < send)
 897                         msg->apply_bytes = 0;
 898                 else
 899                         msg->apply_bytes -= send;
 900                 sk_msg_return_zero(sk, msg, send);
 901                 msg->sg.size -= send;
 902                 release_sock(sk);
 903                 err = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress,
 904                                             &msg_redir, send, flags);
 905                 lock_sock(sk);
 906                 if (err < 0) {
 907                         *copied -= sk_msg_free_nocharge(sk, &msg_redir);
 908                         msg->sg.size = 0;
 909                 }
 910                 if (msg->sg.size == 0)
 911                         tls_free_open_rec(sk);
 912                 break;
 913         case __SK_DROP:
 914         default:
 915                 sk_msg_free_partial(sk, msg, send);
 916                 if (msg->apply_bytes < send)
 917                         msg->apply_bytes = 0;
 918                 else
 919                         msg->apply_bytes -= send;
 920                 if (msg->sg.size == 0)
 921                         tls_free_open_rec(sk);
 922                 *copied -= (send + delta);
 923                 err = -EACCES;
 924         }
 925
 926         if (likely(!err)) {
 927                 bool reset_eval = !ctx->open_rec;
 928
 929                 rec = ctx->open_rec;
 930                 if (rec) {
 931                         msg = &rec->msg_plaintext;
 932                         if (!msg->apply_bytes)
 933                                 reset_eval = true;
 934                 }
 935                 if (reset_eval) {
 936                         psock->eval = __SK_NONE;
 937                         if (psock->sk_redir) {
 938                                 sock_put(psock->sk_redir);
 939                                 psock->sk_redir = NULL;
 940                         }
 941                 }
 942                 if (rec)
 943                         goto more_data;
 944         }
 945  out_err:
 946         sk_psock_put(sk, psock);
 947         return err;
 948 }
 949
 950 static int tls_sw_push_pending_record(struct sock *sk, int flags)
 951 {
 952         struct tls_context *tls_ctx = tls_get_ctx(sk);
 953         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 954         struct tls_rec *rec = ctx->open_rec;
 955         struct sk_msg *msg_pl;
 956         size_t copied;
 957
 958         if (!rec)
 959                 return 0;
 960
 961         msg_pl = &rec->msg_plaintext;
 962         copied = msg_pl->sg.size;
 963         if (!copied)
 964                 return 0;
 965
 966         return bpf_exec_tx_verdict(msg_pl, sk, true, TLS_RECORD_TYPE_DATA,
 967                                    &copied, flags);
 968 }
 969
 970 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 971 {
 972         long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
 973         struct tls_context *tls_ctx = tls_get_ctx(sk);
 974         struct tls_prot_info *prot = &tls_ctx->prot_info;
 975         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
 976         bool async_capable = ctx->async_capable;
 977         unsigned char record_type = TLS_RECORD_TYPE_DATA;
 978         bool is_kvec = iov_iter_is_kvec(&msg->msg_iter);
 979         bool eor = !(msg->msg_flags & MSG_MORE);
 980         size_t try_to_copy;
 981         ssize_t copied = 0;
 982         struct sk_msg *msg_pl, *msg_en;
 983         struct tls_rec *rec;
 984         int required_size;
 985         int num_async = 0;
 986         bool full_record;
 987         int record_room;
 988         int num_zc = 0;
 989         int orig_size;
 990         int ret = 0;
 991
 992         if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
 993                                MSG_CMSG_COMPAT))
 994                 return -EOPNOTSUPP;
 995
 996         ret = mutex_lock_interruptible(&tls_ctx->tx_lock);
 997         if (ret)
 998                 return ret;
 999         lock_sock(sk);
1000
1001         if (unlikely(msg->msg_controllen)) {
1002                 ret = tls_process_cmsg(sk, msg, &record_type);
1003                 if (ret) {
1004                         if (ret == -EINPROGRESS)
1005                                 num_async++;
1006                         else if (ret != -EAGAIN)
1007                                 goto send_end;
1008                 }
1009         }
1010
1011         while (msg_data_left(msg)) {
1012                 if (sk->sk_err) {
1013                         ret = -sk->sk_err;
1014                         goto send_end;
1015                 }
1016
1017                 if (ctx->open_rec)
1018                         rec = ctx->open_rec;
1019                 else
1020                         rec = ctx->open_rec = tls_get_rec(sk);
1021                 if (!rec) {
1022                         ret = -ENOMEM;
1023                         goto send_end;
1024                 }
1025
1026                 msg_pl = &rec->msg_plaintext;
1027                 msg_en = &rec->msg_encrypted;
1028
1029                 orig_size = msg_pl->sg.size;
1030                 full_record = false;
1031                 try_to_copy = msg_data_left(msg);
1032                 record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size;
1033                 if (try_to_copy >= record_room) {
1034                         try_to_copy = record_room;
1035                         full_record = true;
1036                 }
1037
1038                 required_size = msg_pl->sg.size + try_to_copy +
1039                                 prot->overhead_size;
1040
1041                 if (!sk_stream_memory_free(sk))
1042                         goto wait_for_sndbuf;
1043
1044 alloc_encrypted:
1045                 ret = tls_alloc_encrypted_msg(sk, required_size);
1046                 if (ret) {
1047                         if (ret != -ENOSPC)
1048                                 goto wait_for_memory;
1049
1050                         /* Adjust try_to_copy according to the amount that was
1051                          * actually allocated. The difference is due
1052                          * to max sg elements limit
1053                          */
1054                         try_to_copy -= required_size - msg_en->sg.size;
1055                         full_record = true;
1056                 }
1057
1058                 if (!is_kvec && (full_record || eor) && !async_capable) {
1059                         u32 first = msg_pl->sg.end;
1060
1061                         ret = sk_msg_zerocopy_from_iter(sk, &msg->msg_iter,
1062                                                         msg_pl, try_to_copy);
1063                         if (ret)
1064                                 goto fallback_to_reg_send;
1065
1066                         num_zc++;
1067                         copied += try_to_copy;
1068
1069                         sk_msg_sg_copy_set(msg_pl, first);
1070                         ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
1071                                                   record_type, &copied,
1072                                                   msg->msg_flags);
1073                         if (ret) {
1074                                 if (ret == -EINPROGRESS)
1075                                         num_async++;
1076                                 else if (ret == -ENOMEM)
1077                                         goto wait_for_memory;
1078                                 else if (ctx->open_rec && ret == -ENOSPC)
1079                                         goto rollback_iter;
1080                                 else if (ret != -EAGAIN)
1081                                         goto send_end;
1082                         }
1083                         continue;
1084 rollback_iter:
1085                         copied -= try_to_copy;
1086                         sk_msg_sg_copy_clear(msg_pl, first);
1087                         iov_iter_revert(&msg->msg_iter,
1088                                         msg_pl->sg.size - orig_size);
1089 fallback_to_reg_send:
1090                         sk_msg_trim(sk, msg_pl, orig_size);
1091                 }
1092
1093                 required_size = msg_pl->sg.size + try_to_copy;
1094
1095                 ret = tls_clone_plaintext_msg(sk, required_size);
1096                 if (ret) {
1097                         if (ret != -ENOSPC)
1098                                 goto send_end;
1099
1100                         /* Adjust try_to_copy according to the amount that was
1101                          * actually allocated. The difference is due
1102                          * to max sg elements limit
1103                          */
1104                         try_to_copy -= required_size - msg_pl->sg.size;
1105                         full_record = true;
1106                         sk_msg_trim(sk, msg_en,
1107                                     msg_pl->sg.size + prot->overhead_size);
1108                 }
1109
1110                 if (try_to_copy) {
1111                         ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter,
1112                                                        msg_pl, try_to_copy);
1113                         if (ret < 0)
1114                                 goto trim_sgl;
1115                 }
1116
1117                 /* Open records defined only if successfully copied, otherwise
1118                  * we would trim the sg but not reset the open record frags.
1119                  */
1120                 tls_ctx->pending_open_record_frags = true;
1121                 copied += try_to_copy;
1122                 if (full_record || eor) {
1123                         ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
1124                                                   record_type, &copied,
1125                                                   msg->msg_flags);
1126                         if (ret) {
1127                                 if (ret == -EINPROGRESS)
1128                                         num_async++;
1129                                 else if (ret == -ENOMEM)
1130                                         goto wait_for_memory;
1131                                 else if (ret != -EAGAIN) {
1132                                         if (ret == -ENOSPC)
1133                                                 ret = 0;
1134                                         goto send_end;
1135                                 }
1136                         }
1137                 }
1138
1139                 continue;
1140
1141 wait_for_sndbuf:
1142                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1143 wait_for_memory:
1144                 ret = sk_stream_wait_memory(sk, &timeo);
1145                 if (ret) {
1146 trim_sgl:
1147                         if (ctx->open_rec)
1148                                 tls_trim_both_msgs(sk, orig_size);
1149                         goto send_end;
1150                 }
1151
1152                 if (ctx->open_rec && msg_en->sg.size < required_size)
1153                         goto alloc_encrypted;
1154         }
1155
1156         if (!num_async) {
1157                 goto send_end;
1158         } else if (num_zc) {
1159                 int err;
1160
1161                 /* Wait for pending encryptions to get completed */
1162                 err = tls_encrypt_async_wait(ctx);
1163                 if (err) {
1164                         ret = err;
1165                         copied = 0;
1166                 }
1167         }
1168
1169         /* Transmit if any encryptions have completed */
1170         if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
1171                 cancel_delayed_work(&ctx->tx_work.work);
1172                 tls_tx_records(sk, msg->msg_flags);
1173         }
1174
1175 send_end:
1176         ret = sk_stream_error(sk, msg->msg_flags, ret);
1177
1178         release_sock(sk);
1179         mutex_unlock(&tls_ctx->tx_lock);
1180         return copied > 0 ? copied : ret;
1181 }
1182
1183 /*
1184  * Handle unexpected EOF during splice without SPLICE_F_MORE set.
1185  */
1186 void tls_sw_splice_eof(struct socket *sock)
1187 {
1188         struct sock *sk = sock->sk;
1189         struct tls_context *tls_ctx = tls_get_ctx(sk);
1190         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
1191         struct tls_rec *rec;
1192         struct sk_msg *msg_pl;
1193         ssize_t copied = 0;
1194         bool retrying = false;
1195         int ret = 0;
1196
1197         if (!ctx->open_rec)
1198                 return;
1199
1200         mutex_lock(&tls_ctx->tx_lock);
1201         lock_sock(sk);
1202
1203 retry:
1204         /* same checks as in tls_sw_push_pending_record() */
1205         rec = ctx->open_rec;
1206         if (!rec)
1207                 goto unlock;
1208
1209         msg_pl = &rec->msg_plaintext;
1210         if (msg_pl->sg.size == 0)
1211                 goto unlock;
1212
1213         /* Check the BPF advisor and perform transmission. */
1214         ret = bpf_exec_tx_verdict(msg_pl, sk, false, TLS_RECORD_TYPE_DATA,
1215                                   &copied, 0);
1216         switch (ret) {
1217         case 0:
1218         case -EAGAIN:
1219                 if (retrying)
1220                         goto unlock;
1221                 retrying = true;
1222                 goto retry;
1223         case -EINPROGRESS:
1224                 break;
1225         default:
1226                 goto unlock;
1227         }
1228
1229         /* Wait for pending encryptions to get completed */
1230         if (tls_encrypt_async_wait(ctx))
1231                 goto unlock;
1232
1233         /* Transmit if any encryptions have completed */
1234         if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
1235                 cancel_delayed_work(&ctx->tx_work.work);
1236                 tls_tx_records(sk, 0);
1237         }
1238
1239 unlock:
1240         release_sock(sk);
1241         mutex_unlock(&tls_ctx->tx_lock);
1242 }
1243
1244 static int tls_sw_do_sendpage(struct sock *sk, struct page *page,
1245                               int offset, size_t size, int flags)
1246 {
1247         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1248         struct tls_context *tls_ctx = tls_get_ctx(sk);
1249         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
1250         struct tls_prot_info *prot = &tls_ctx->prot_info;
1251         unsigned char record_type = TLS_RECORD_TYPE_DATA;
1252         struct sk_msg *msg_pl;
1253         struct tls_rec *rec;
1254         int num_async = 0;
1255         ssize_t copied = 0;
1256         bool full_record;
1257         int record_room;
1258         int ret = 0;
1259         bool eor;
1260
1261         eor = !(flags & MSG_SENDPAGE_NOTLAST);
1262         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1263
1264         /* Call the sk_stream functions to manage the sndbuf mem. */
1265         while (size > 0) {
1266                 size_t copy, required_size;
1267
1268                 if (sk->sk_err) {
1269                         ret = -sk->sk_err;
1270                         goto sendpage_end;
1271                 }
1272
1273                 if (ctx->open_rec)
1274                         rec = ctx->open_rec;
1275                 else
1276                         rec = ctx->open_rec = tls_get_rec(sk);
1277                 if (!rec) {
1278                         ret = -ENOMEM;
1279                         goto sendpage_end;
1280                 }
1281
1282                 msg_pl = &rec->msg_plaintext;
1283
1284                 full_record = false;
1285                 record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size;
1286                 copy = size;
1287                 if (copy >= record_room) {
1288                         copy = record_room;
1289                         full_record = true;
1290                 }
1291
1292                 required_size = msg_pl->sg.size + copy + prot->overhead_size;
1293
1294                 if (!sk_stream_memory_free(sk))
1295                         goto wait_for_sndbuf;
1296 alloc_payload:
1297                 ret = tls_alloc_encrypted_msg(sk, required_size);
1298                 if (ret) {
1299                         if (ret != -ENOSPC)
1300                                 goto wait_for_memory;
1301
1302                         /* Adjust copy according to the amount that was
1303                          * actually allocated. The difference is due
1304                          * to max sg elements limit
1305                          */
1306                         copy -= required_size - msg_pl->sg.size;
1307                         full_record = true;
1308                 }
1309
1310                 sk_msg_page_add(msg_pl, page, copy, offset);
1311                 msg_pl->sg.copybreak = 0;
1312                 msg_pl->sg.curr = msg_pl->sg.end;
1313                 sk_mem_charge(sk, copy);
1314
1315                 offset += copy;
1316                 size -= copy;
1317                 copied += copy;
1318
1319                 tls_ctx->pending_open_record_frags = true;
1320                 if (full_record || eor || sk_msg_full(msg_pl)) {
1321                         ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
1322                                                   record_type, &copied, flags);
1323                         if (ret) {
1324                                 if (ret == -EINPROGRESS)
1325                                         num_async++;
1326                                 else if (ret == -ENOMEM)
1327                                         goto wait_for_memory;
1328                                 else if (ret != -EAGAIN) {
1329                                         if (ret == -ENOSPC)
1330                                                 ret = 0;
1331                                         goto sendpage_end;
1332                                 }
1333                         }
1334                 }
1335                 continue;
1336 wait_for_sndbuf:
1337                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1338 wait_for_memory:
1339                 ret = sk_stream_wait_memory(sk, &timeo);
1340                 if (ret) {
1341                         if (ctx->open_rec)
1342                                 tls_trim_both_msgs(sk, msg_pl->sg.size);
1343                         goto sendpage_end;
1344                 }
1345
1346                 if (ctx->open_rec)
1347                         goto alloc_payload;
1348         }
1349
1350         if (num_async) {
1351                 /* Transmit if any encryptions have completed */
1352                 if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
1353                         cancel_delayed_work(&ctx->tx_work.work);
1354                         tls_tx_records(sk, flags);
1355                 }
1356         }
1357 sendpage_end:
1358         ret = sk_stream_error(sk, flags, ret);
1359         return copied > 0 ? copied : ret;
1360 }
1361
1362 int tls_sw_sendpage_locked(struct sock *sk, struct page *page,
1363                            int offset, size_t size, int flags)
1364 {
1365         if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
1366                       MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY |
1367                       MSG_NO_SHARED_FRAGS))
1368                 return -EOPNOTSUPP;
1369
1370         return tls_sw_do_sendpage(sk, page, offset, size, flags);
1371 }
1372
1373 int tls_sw_sendpage(struct sock *sk, struct page *page,
1374                     int offset, size_t size, int flags)
1375 {
1376         struct tls_context *tls_ctx = tls_get_ctx(sk);
1377         int ret;
1378
1379         if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
1380                       MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY))
1381                 return -EOPNOTSUPP;
1382
1383         ret = mutex_lock_interruptible(&tls_ctx->tx_lock);
1384         if (ret)
1385                 return ret;
1386         lock_sock(sk);
1387         ret = tls_sw_do_sendpage(sk, page, offset, size, flags);
1388         release_sock(sk);
1389         mutex_unlock(&tls_ctx->tx_lock);
1390         return ret;
1391 }
1392
1393 static int
1394 tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock,
1395                 bool released)
1396 {
1397         struct tls_context *tls_ctx = tls_get_ctx(sk);
1398         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1399         DEFINE_WAIT_FUNC(wait, woken_wake_function);
1400         int ret = 0;
1401         long timeo;
1402
1403         timeo = sock_rcvtimeo(sk, nonblock);
1404
1405         while (!tls_strp_msg_ready(ctx)) {
1406                 if (!sk_psock_queue_empty(psock))
1407                         return 0;
1408
1409                 if (sk->sk_err)
1410                         return sock_error(sk);
1411
1412                 if (ret < 0)
1413                         return ret;
1414
1415                 if (!skb_queue_empty(&sk->sk_receive_queue)) {
1416                         tls_strp_check_rcv(&ctx->strp);
1417                         if (tls_strp_msg_ready(ctx))
1418                                 break;
1419                 }
1420
1421                 if (sk->sk_shutdown & RCV_SHUTDOWN)
1422                         return 0;
1423
1424                 if (sock_flag(sk, SOCK_DONE))
1425                         return 0;
1426
1427                 if (!timeo)
1428                         return -EAGAIN;
1429
1430                 released = true;
1431                 add_wait_queue(sk_sleep(sk), &wait);
1432                 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1433                 ret = sk_wait_event(sk, &timeo,
1434                                     tls_strp_msg_ready(ctx) ||
1435                                     !sk_psock_queue_empty(psock),
1436                                     &wait);
1437                 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1438                 remove_wait_queue(sk_sleep(sk), &wait);
1439
1440                 /* Handle signals */
1441                 if (signal_pending(current))
1442                         return sock_intr_errno(timeo);
1443         }
1444
1445         tls_strp_msg_load(&ctx->strp, released);
1446
1447         return 1;
1448 }
1449
1450 static int tls_setup_from_iter(struct iov_iter *from,
1451                                int length, int *pages_used,
1452                                struct scatterlist *to,
1453                                int to_max_pages)
1454 {
1455         int rc = 0, i = 0, num_elem = *pages_used, maxpages;
1456         struct page *pages[MAX_SKB_FRAGS];
1457         unsigned int size = 0;
1458         ssize_t copied, use;
1459         size_t offset;
1460
1461         while (length > 0) {
1462                 i = 0;
1463                 maxpages = to_max_pages - num_elem;
1464                 if (maxpages == 0) {
1465                         rc = -EFAULT;
1466                         goto out;
1467                 }
1468                 copied = iov_iter_get_pages2(from, pages,
1469                                             length,
1470                                             maxpages, &offset);
1471                 if (copied <= 0) {
1472                         rc = -EFAULT;
1473                         goto out;
1474                 }
1475
1476                 length -= copied;
1477                 size += copied;
1478                 while (copied) {
1479                         use = min_t(int, copied, PAGE_SIZE - offset);
1480
1481                         sg_set_page(&to[num_elem],
1482                                     pages[i], use, offset);
1483                         sg_unmark_end(&to[num_elem]);
1484                         /* We do not uncharge memory from this API */
1485
1486                         offset = 0;
1487                         copied -= use;
1488
1489                         i++;
1490                         num_elem++;
1491                 }
1492         }
1493         /* Mark the end in the last sg entry if newly added */
1494         if (num_elem > *pages_used)
1495                 sg_mark_end(&to[num_elem - 1]);
1496 out:
1497         if (rc)
1498                 iov_iter_revert(from, size);
1499         *pages_used = num_elem;
1500
1501         return rc;
1502 }
1503
1504 static struct sk_buff *
1505 tls_alloc_clrtxt_skb(struct sock *sk, struct sk_buff *skb,
1506                      unsigned int full_len)
1507 {
1508         struct strp_msg *clr_rxm;
1509         struct sk_buff *clr_skb;
1510         int err;
1511
1512         clr_skb = alloc_skb_with_frags(0, full_len, TLS_PAGE_ORDER,
1513                                        &err, sk->sk_allocation);
1514         if (!clr_skb)
1515                 return NULL;
1516
1517         skb_copy_header(clr_skb, skb);
1518         clr_skb->len = full_len;
1519         clr_skb->data_len = full_len;
1520
1521         clr_rxm = strp_msg(clr_skb);
1522         clr_rxm->offset = 0;
1523
1524         return clr_skb;
1525 }
1526
1527 /* Decrypt handlers
1528  *
1529  * tls_decrypt_sw() and tls_decrypt_device() are decrypt handlers.
1530  * They must transform the darg in/out argument are as follows:
1531  *       |          Input            |         Output
1532  * -------------------------------------------------------------------
1533  *    zc | Zero-copy decrypt allowed | Zero-copy performed
1534  * async | Async decrypt allowed     | Async crypto used / in progress
1535  *   skb |            *              | Output skb
1536  *
1537  * If ZC decryption was performed darg.skb will point to the input skb.
1538  */
1539
1540 /* This function decrypts the input skb into either out_iov or in out_sg
1541  * or in skb buffers itself. The input parameter 'darg->zc' indicates if
1542  * zero-copy mode needs to be tried or not. With zero-copy mode, either
1543  * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are
1544  * NULL, then the decryption happens inside skb buffers itself, i.e.
1545  * zero-copy gets disabled and 'darg->zc' is updated.
1546  */
1547 static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov,
1548                           struct scatterlist *out_sg,
1549                           struct tls_decrypt_arg *darg)
1550 {
1551         struct tls_context *tls_ctx = tls_get_ctx(sk);
1552         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1553         struct tls_prot_info *prot = &tls_ctx->prot_info;
1554         int n_sgin, n_sgout, aead_size, err, pages = 0;
1555         struct sk_buff *skb = tls_strp_msg(ctx);
1556         const struct strp_msg *rxm = strp_msg(skb);
1557         const struct tls_msg *tlm = tls_msg(skb);
1558         struct aead_request *aead_req;
1559         struct scatterlist *sgin = NULL;
1560         struct scatterlist *sgout = NULL;
1561         const int data_len = rxm->full_len - prot->overhead_size;
1562         int tail_pages = !!prot->tail_size;
1563         struct tls_decrypt_ctx *dctx;
1564         struct sk_buff *clear_skb;
1565         int iv_offset = 0;
1566         u8 *mem;
1567
1568         n_sgin = skb_nsg(skb, rxm->offset + prot->prepend_size,
1569                          rxm->full_len - prot->prepend_size);
1570         if (n_sgin < 1)
1571                 return n_sgin ?: -EBADMSG;
1572
1573         if (darg->zc && (out_iov || out_sg)) {
1574                 clear_skb = NULL;
1575
1576                 if (out_iov)
1577                         n_sgout = 1 + tail_pages +
1578                                 iov_iter_npages_cap(out_iov, INT_MAX, data_len);
1579                 else
1580                         n_sgout = sg_nents(out_sg);
1581         } else {
1582                 darg->zc = false;
1583
1584                 clear_skb = tls_alloc_clrtxt_skb(sk, skb, rxm->full_len);
1585                 if (!clear_skb)
1586                         return -ENOMEM;
1587
1588                 n_sgout = 1 + skb_shinfo(clear_skb)->nr_frags;
1589         }
1590
1591         /* Increment to accommodate AAD */
1592         n_sgin = n_sgin + 1;
1593
1594         /* Allocate a single block of memory which contains
1595          *   aead_req || tls_decrypt_ctx.
1596          * Both structs are variable length.
1597          */
1598         aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv);
1599         aead_size = ALIGN(aead_size, __alignof__(*dctx));
1600         mem = kmalloc(aead_size + struct_size(dctx, sg, size_add(n_sgin, n_sgout)),
1601                       sk->sk_allocation);
1602         if (!mem) {
1603                 err = -ENOMEM;
1604                 goto exit_free_skb;
1605         }
1606
1607         /* Segment the allocated memory */
1608         aead_req = (struct aead_request *)mem;
1609         dctx = (struct tls_decrypt_ctx *)(mem + aead_size);
1610         dctx->sk = sk;
1611         sgin = &dctx->sg[0];
1612         sgout = &dctx->sg[n_sgin];
1613
1614         /* For CCM based ciphers, first byte of nonce+iv is a constant */
1615         switch (prot->cipher_type) {
1616         case TLS_CIPHER_AES_CCM_128:
1617                 dctx->iv[0] = TLS_AES_CCM_IV_B0_BYTE;
1618                 iv_offset = 1;
1619                 break;
1620         case TLS_CIPHER_SM4_CCM:
1621                 dctx->iv[0] = TLS_SM4_CCM_IV_B0_BYTE;
1622                 iv_offset = 1;
1623                 break;
1624         }
1625
1626         /* Prepare IV */
1627         if (prot->version == TLS_1_3_VERSION ||
1628             prot->cipher_type == TLS_CIPHER_CHACHA20_POLY1305) {
1629                 memcpy(&dctx->iv[iv_offset], tls_ctx->rx.iv,
1630                        prot->iv_size + prot->salt_size);
1631         } else {
1632                 err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE,
1633                                     &dctx->iv[iv_offset] + prot->salt_size,
1634                                     prot->iv_size);
1635                 if (err < 0)
1636                         goto exit_free;
1637                 memcpy(&dctx->iv[iv_offset], tls_ctx->rx.iv, prot->salt_size);
1638         }
1639         tls_xor_iv_with_seq(prot, &dctx->iv[iv_offset], tls_ctx->rx.rec_seq);
1640
1641         /* Prepare AAD */
1642         tls_make_aad(dctx->aad, rxm->full_len - prot->overhead_size +
1643                      prot->tail_size,
1644                      tls_ctx->rx.rec_seq, tlm->control, prot);
1645
1646         /* Prepare sgin */
1647         sg_init_table(sgin, n_sgin);
1648         sg_set_buf(&sgin[0], dctx->aad, prot->aad_size);
1649         err = skb_to_sgvec(skb, &sgin[1],
1650                            rxm->offset + prot->prepend_size,
1651                            rxm->full_len - prot->prepend_size);
1652         if (err < 0)
1653                 goto exit_free;
1654
1655         if (clear_skb) {
1656                 sg_init_table(sgout, n_sgout);
1657                 sg_set_buf(&sgout[0], dctx->aad, prot->aad_size);
1658
1659                 err = skb_to_sgvec(clear_skb, &sgout[1], prot->prepend_size,
1660                                    data_len + prot->tail_size);
1661                 if (err < 0)
1662                         goto exit_free;
1663         } else if (out_iov) {
1664                 sg_init_table(sgout, n_sgout);
1665                 sg_set_buf(&sgout[0], dctx->aad, prot->aad_size);
1666
1667                 err = tls_setup_from_iter(out_iov, data_len, &pages, &sgout[1],
1668                                           (n_sgout - 1 - tail_pages));
1669                 if (err < 0)
1670                         goto exit_free_pages;
1671
1672                 if (prot->tail_size) {
1673                         sg_unmark_end(&sgout[pages]);
1674                         sg_set_buf(&sgout[pages + 1], &dctx->tail,
1675                                    prot->tail_size);
1676                         sg_mark_end(&sgout[pages + 1]);
1677                 }
1678         } else if (out_sg) {
1679                 memcpy(sgout, out_sg, n_sgout * sizeof(*sgout));
1680         }
1681         dctx->free_sgout = !!pages;
1682
1683         /* Prepare and submit AEAD request */
1684         err = tls_do_decryption(sk, sgin, sgout, dctx->iv,
1685                                 data_len + prot->tail_size, aead_req, darg);
1686         if (err) {
1687                 if (darg->async_done)
1688                         goto exit_free_skb;
1689                 goto exit_free_pages;
1690         }
1691
1692         darg->skb = clear_skb ?: tls_strp_msg(ctx);
1693         clear_skb = NULL;
1694
1695         if (unlikely(darg->async)) {
1696                 err = tls_strp_msg_hold(&ctx->strp, &ctx->async_hold);
1697                 if (err)
1698                         __skb_queue_tail(&ctx->async_hold, darg->skb);
1699                 return err;
1700         }
1701
1702         if (unlikely(darg->async_done))
1703                 return 0;
1704
1705         if (prot->tail_size)
1706                 darg->tail = dctx->tail;
1707
1708 exit_free_pages:
1709         /* Release the pages in case iov was mapped to pages */
1710         for (; pages > 0; pages--)
1711                 put_page(sg_page(&sgout[pages]));
1712 exit_free:
1713         kfree(mem);
1714 exit_free_skb:
1715         consume_skb(clear_skb);
1716         return err;
1717 }
1718
1719 static int
1720 tls_decrypt_sw(struct sock *sk, struct tls_context *tls_ctx,
1721                struct msghdr *msg, struct tls_decrypt_arg *darg)
1722 {
1723         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1724         struct tls_prot_info *prot = &tls_ctx->prot_info;
1725         struct strp_msg *rxm;
1726         int pad, err;
1727
1728         err = tls_decrypt_sg(sk, &msg->msg_iter, NULL, darg);
1729         if (err < 0) {
1730                 if (err == -EBADMSG)
1731                         TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTERROR);
1732                 return err;
1733         }
1734         /* keep going even for ->async, the code below is TLS 1.3 */
1735
1736         /* If opportunistic TLS 1.3 ZC failed retry without ZC */
1737         if (unlikely(darg->zc && prot->version == TLS_1_3_VERSION &&
1738                      darg->tail != TLS_RECORD_TYPE_DATA)) {
1739                 darg->zc = false;
1740                 if (!darg->tail)
1741                         TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXNOPADVIOL);
1742                 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTRETRY);
1743                 return tls_decrypt_sw(sk, tls_ctx, msg, darg);
1744         }
1745
1746         pad = tls_padding_length(prot, darg->skb, darg);
1747         if (pad < 0) {
1748                 if (darg->skb != tls_strp_msg(ctx))
1749                         consume_skb(darg->skb);
1750                 return pad;
1751         }
1752
1753         rxm = strp_msg(darg->skb);
1754         rxm->full_len -= pad;
1755
1756         return 0;
1757 }
1758
1759 static int
1760 tls_decrypt_device(struct sock *sk, struct msghdr *msg,
1761                    struct tls_context *tls_ctx, struct tls_decrypt_arg *darg)
1762 {
1763         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1764         struct tls_prot_info *prot = &tls_ctx->prot_info;
1765         struct strp_msg *rxm;
1766         int pad, err;
1767
1768         if (tls_ctx->rx_conf != TLS_HW)
1769                 return 0;
1770
1771         err = tls_device_decrypted(sk, tls_ctx);
1772         if (err <= 0)
1773                 return err;
1774
1775         pad = tls_padding_length(prot, tls_strp_msg(ctx), darg);
1776         if (pad < 0)
1777                 return pad;
1778
1779         darg->async = false;
1780         darg->skb = tls_strp_msg(ctx);
1781         /* ->zc downgrade check, in case TLS 1.3 gets here */
1782         darg->zc &= !(prot->version == TLS_1_3_VERSION &&
1783                       tls_msg(darg->skb)->control != TLS_RECORD_TYPE_DATA);
1784
1785         rxm = strp_msg(darg->skb);
1786         rxm->full_len -= pad;
1787
1788         if (!darg->zc) {
1789                 /* Non-ZC case needs a real skb */
1790                 darg->skb = tls_strp_msg_detach(ctx);
1791                 if (!darg->skb)
1792                         return -ENOMEM;
1793         } else {
1794                 unsigned int off, len;
1795
1796                 /* In ZC case nobody cares about the output skb.
1797                  * Just copy the data here. Note the skb is not fully trimmed.
1798                  */
1799                 off = rxm->offset + prot->prepend_size;
1800                 len = rxm->full_len - prot->overhead_size;
1801
1802                 err = skb_copy_datagram_msg(darg->skb, off, msg, len);
1803                 if (err)
1804                         return err;
1805         }
1806         return 1;
1807 }
1808
1809 static int tls_rx_one_record(struct sock *sk, struct msghdr *msg,
1810                              struct tls_decrypt_arg *darg)
1811 {
1812         struct tls_context *tls_ctx = tls_get_ctx(sk);
1813         struct tls_prot_info *prot = &tls_ctx->prot_info;
1814         struct strp_msg *rxm;
1815         int err;
1816
1817         err = tls_decrypt_device(sk, msg, tls_ctx, darg);
1818         if (!err)
1819                 err = tls_decrypt_sw(sk, tls_ctx, msg, darg);
1820         if (err < 0)
1821                 return err;
1822
1823         rxm = strp_msg(darg->skb);
1824         rxm->offset += prot->prepend_size;
1825         rxm->full_len -= prot->overhead_size;
1826         tls_advance_record_sn(sk, prot, &tls_ctx->rx);
1827
1828         return 0;
1829 }
1830
1831 int decrypt_skb(struct sock *sk, struct scatterlist *sgout)
1832 {
1833         struct tls_decrypt_arg darg = { .zc = true, };
1834
1835         return tls_decrypt_sg(sk, NULL, sgout, &darg);
1836 }
1837
1838 static int tls_record_content_type(struct msghdr *msg, struct tls_msg *tlm,
1839                                    u8 *control)
1840 {
1841         int err;
1842
1843         if (!*control) {
1844                 *control = tlm->control;
1845                 if (!*control)
1846                         return -EBADMSG;
1847
1848                 err = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
1849                                sizeof(*control), control);
1850                 if (*control != TLS_RECORD_TYPE_DATA) {
1851                         if (err || msg->msg_flags & MSG_CTRUNC)
1852                                 return -EIO;
1853                 }
1854         } else if (*control != tlm->control) {
1855                 return 0;
1856         }
1857
1858         return 1;
1859 }
1860
1861 static void tls_rx_rec_done(struct tls_sw_context_rx *ctx)
1862 {
1863         tls_strp_msg_done(&ctx->strp);
1864 }
1865
1866 /* This function traverses the rx_list in tls receive context to copies the
1867  * decrypted records into the buffer provided by caller zero copy is not
1868  * true. Further, the records are removed from the rx_list if it is not a peek
1869  * case and the record has been consumed completely.
1870  */
1871 static int process_rx_list(struct tls_sw_context_rx *ctx,
1872                            struct msghdr *msg,
1873                            u8 *control,
1874                            size_t skip,
1875                            size_t len,
1876                            bool is_peek,
1877                            bool *more)
1878 {
1879         struct sk_buff *skb = skb_peek(&ctx->rx_list);
1880         struct tls_msg *tlm;
1881         ssize_t copied = 0;
1882         int err;
1883
1884         while (skip && skb) {
1885                 struct strp_msg *rxm = strp_msg(skb);
1886                 tlm = tls_msg(skb);
1887
1888                 err = tls_record_content_type(msg, tlm, control);
1889                 if (err <= 0)
1890                         goto more;
1891
1892                 if (skip < rxm->full_len)
1893                         break;
1894
1895                 skip = skip - rxm->full_len;
1896                 skb = skb_peek_next(skb, &ctx->rx_list);
1897         }
1898
1899         while (len && skb) {
1900                 struct sk_buff *next_skb;
1901                 struct strp_msg *rxm = strp_msg(skb);
1902                 int chunk = min_t(unsigned int, rxm->full_len - skip, len);
1903
1904                 tlm = tls_msg(skb);
1905
1906                 err = tls_record_content_type(msg, tlm, control);
1907                 if (err <= 0)
1908                         goto more;
1909
1910                 err = skb_copy_datagram_msg(skb, rxm->offset + skip,
1911                                             msg, chunk);
1912                 if (err < 0)
1913                         goto more;
1914
1915                 len = len - chunk;
1916                 copied = copied + chunk;
1917
1918                 /* Consume the data from record if it is non-peek case*/
1919                 if (!is_peek) {
1920                         rxm->offset = rxm->offset + chunk;
1921                         rxm->full_len = rxm->full_len - chunk;
1922
1923                         /* Return if there is unconsumed data in the record */
1924                         if (rxm->full_len - skip)
1925                                 break;
1926                 }
1927
1928                 /* The remaining skip-bytes must lie in 1st record in rx_list.
1929                  * So from the 2nd record, 'skip' should be 0.
1930                  */
1931                 skip = 0;
1932
1933                 if (msg)
1934                         msg->msg_flags |= MSG_EOR;
1935
1936                 next_skb = skb_peek_next(skb, &ctx->rx_list);
1937
1938                 if (!is_peek) {
1939                         __skb_unlink(skb, &ctx->rx_list);
1940                         consume_skb(skb);
1941                 }
1942
1943                 skb = next_skb;
1944         }
1945         err = 0;
1946
1947 out:
1948         return copied ? : err;
1949 more:
1950         if (more)
1951                 *more = true;
1952         goto out;
1953 }
1954
1955 static bool
1956 tls_read_flush_backlog(struct sock *sk, struct tls_prot_info *prot,
1957                        size_t len_left, size_t decrypted, ssize_t done,
1958                        size_t *flushed_at)
1959 {
1960         size_t max_rec;
1961
1962         if (len_left <= decrypted)
1963                 return false;
1964
1965         max_rec = prot->overhead_size - prot->tail_size + TLS_MAX_PAYLOAD_SIZE;
1966         if (done - *flushed_at < SZ_128K && tcp_inq(sk) > max_rec)
1967                 return false;
1968
1969         *flushed_at = done;
1970         return sk_flush_backlog(sk);
1971 }
1972
1973 static int tls_rx_reader_acquire(struct sock *sk, struct tls_sw_context_rx *ctx,
1974                                  bool nonblock)
1975 {
1976         long timeo;
1977         int ret;
1978
1979         timeo = sock_rcvtimeo(sk, nonblock);
1980
1981         while (unlikely(ctx->reader_present)) {
1982                 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1983
1984                 ctx->reader_contended = 1;
1985
1986                 add_wait_queue(&ctx->wq, &wait);
1987                 ret = sk_wait_event(sk, &timeo,
1988                                     !READ_ONCE(ctx->reader_present), &wait);
1989                 remove_wait_queue(&ctx->wq, &wait);
1990
1991                 if (timeo <= 0)
1992                         return -EAGAIN;
1993                 if (signal_pending(current))
1994                         return sock_intr_errno(timeo);
1995                 if (ret < 0)
1996                         return ret;
1997         }
1998
1999         WRITE_ONCE(ctx->reader_present, 1);
2000
2001         return 0;
2002 }
2003
2004 static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx,
2005                               bool nonblock)
2006 {
2007         int err;
2008
2009         lock_sock(sk);
2010         err = tls_rx_reader_acquire(sk, ctx, nonblock);
2011         if (err)
2012                 release_sock(sk);
2013         return err;
2014 }
2015
2016 static void tls_rx_reader_release(struct sock *sk, struct tls_sw_context_rx *ctx)
2017 {
2018         if (unlikely(ctx->reader_contended)) {
2019                 if (wq_has_sleeper(&ctx->wq))
2020                         wake_up(&ctx->wq);
2021                 else
2022                         ctx->reader_contended = 0;
2023
2024                 WARN_ON_ONCE(!ctx->reader_present);
2025         }
2026
2027         WRITE_ONCE(ctx->reader_present, 0);
2028 }
2029
2030 static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx)
2031 {
2032         tls_rx_reader_release(sk, ctx);
2033         release_sock(sk);
2034 }
2035
2036 int tls_sw_recvmsg(struct sock *sk,
2037                    struct msghdr *msg,
2038                    size_t len,
2039                    int flags,
2040                    int *addr_len)
2041 {
2042         struct tls_context *tls_ctx = tls_get_ctx(sk);
2043         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2044         struct tls_prot_info *prot = &tls_ctx->prot_info;
2045         ssize_t decrypted = 0, async_copy_bytes = 0;
2046         struct sk_psock *psock;
2047         unsigned char control = 0;
2048         size_t flushed_at = 0;
2049         struct strp_msg *rxm;
2050         struct tls_msg *tlm;
2051         ssize_t copied = 0;
2052         ssize_t peeked = 0;
2053         bool async = false;
2054         int target, err;
2055         bool is_kvec = iov_iter_is_kvec(&msg->msg_iter);
2056         bool is_peek = flags & MSG_PEEK;
2057         bool rx_more = false;
2058         bool released = true;
2059         bool bpf_strp_enabled;
2060         bool zc_capable;
2061
2062         if (unlikely(flags & MSG_ERRQUEUE))
2063                 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR);
2064
2065         err = tls_rx_reader_lock(sk, ctx, flags & MSG_DONTWAIT);
2066         if (err < 0)
2067                 return err;
2068         psock = sk_psock_get(sk);
2069         bpf_strp_enabled = sk_psock_strp_enabled(psock);
2070
2071         /* If crypto failed the connection is broken */
2072         err = ctx->async_wait.err;
2073         if (err)
2074                 goto end;
2075
2076         /* Process pending decrypted records. It must be non-zero-copy */
2077         err = process_rx_list(ctx, msg, &control, 0, len, is_peek, &rx_more);
2078         if (err < 0)
2079                 goto end;
2080
2081         copied = err;
2082         if (len <= copied || (copied && control != TLS_RECORD_TYPE_DATA) || rx_more)
2083                 goto end;
2084
2085         target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
2086         len = len - copied;
2087
2088         zc_capable = !bpf_strp_enabled && !is_kvec && !is_peek &&
2089                 ctx->zc_capable;
2090         decrypted = 0;
2091         while (len && (decrypted + copied < target || tls_strp_msg_ready(ctx))) {
2092                 struct tls_decrypt_arg darg;
2093                 int to_decrypt, chunk;
2094
2095                 err = tls_rx_rec_wait(sk, psock, flags & MSG_DONTWAIT,
2096                                       released);
2097                 if (err <= 0) {
2098                         if (psock) {
2099                                 chunk = sk_msg_recvmsg(sk, psock, msg, len,
2100                                                        flags);
2101                                 if (chunk > 0) {
2102                                         decrypted += chunk;
2103                                         len -= chunk;
2104                                         continue;
2105                                 }
2106                         }
2107                         goto recv_end;
2108                 }
2109
2110                 memset(&darg.inargs, 0, sizeof(darg.inargs));
2111
2112                 rxm = strp_msg(tls_strp_msg(ctx));
2113                 tlm = tls_msg(tls_strp_msg(ctx));
2114
2115                 to_decrypt = rxm->full_len - prot->overhead_size;
2116
2117                 if (zc_capable && to_decrypt <= len &&
2118                     tlm->control == TLS_RECORD_TYPE_DATA)
2119                         darg.zc = true;
2120
2121                 /* Do not use async mode if record is non-data */
2122                 if (tlm->control == TLS_RECORD_TYPE_DATA && !bpf_strp_enabled)
2123                         darg.async = ctx->async_capable;
2124                 else
2125                         darg.async = false;
2126
2127                 err = tls_rx_one_record(sk, msg, &darg);
2128                 if (err < 0) {
2129                         tls_err_abort(sk, -EBADMSG);
2130                         goto recv_end;
2131                 }
2132
2133                 async |= darg.async;
2134
2135                 /* If the type of records being processed is not known yet,
2136                  * set it to record type just dequeued. If it is already known,
2137                  * but does not match the record type just dequeued, go to end.
2138                  * We always get record type here since for tls1.2, record type
2139                  * is known just after record is dequeued from stream parser.
2140                  * For tls1.3, we disable async.
2141                  */
2142                 err = tls_record_content_type(msg, tls_msg(darg.skb), &control);
2143                 if (err <= 0) {
2144                         DEBUG_NET_WARN_ON_ONCE(darg.zc);
2145                         tls_rx_rec_done(ctx);
2146 put_on_rx_list_err:
2147                         __skb_queue_tail(&ctx->rx_list, darg.skb);
2148                         goto recv_end;
2149                 }
2150
2151                 /* periodically flush backlog, and feed strparser */
2152                 released = tls_read_flush_backlog(sk, prot, len, to_decrypt,
2153                                                   decrypted + copied,
2154                                                   &flushed_at);
2155
2156                 /* TLS 1.3 may have updated the length by more than overhead */
2157                 rxm = strp_msg(darg.skb);
2158                 chunk = rxm->full_len;
2159                 tls_rx_rec_done(ctx);
2160
2161                 if (!darg.zc) {
2162                         bool partially_consumed = chunk > len;
2163                         struct sk_buff *skb = darg.skb;
2164
2165                         DEBUG_NET_WARN_ON_ONCE(darg.skb == ctx->strp.anchor);
2166
2167                         if (async) {
2168                                 /* TLS 1.2-only, to_decrypt must be text len */
2169                                 chunk = min_t(int, to_decrypt, len);
2170                                 async_copy_bytes += chunk;
2171 put_on_rx_list:
2172                                 decrypted += chunk;
2173                                 len -= chunk;
2174                                 __skb_queue_tail(&ctx->rx_list, skb);
2175                                 if (unlikely(control != TLS_RECORD_TYPE_DATA))
2176                                         break;
2177                                 continue;
2178                         }
2179
2180                         if (bpf_strp_enabled) {
2181                                 released = true;
2182                                 err = sk_psock_tls_strp_read(psock, skb);
2183                                 if (err != __SK_PASS) {
2184                                         rxm->offset = rxm->offset + rxm->full_len;
2185                                         rxm->full_len = 0;
2186                                         if (err == __SK_DROP)
2187                                                 consume_skb(skb);
2188                                         continue;
2189                                 }
2190                         }
2191
2192                         if (partially_consumed)
2193                                 chunk = len;
2194
2195                         err = skb_copy_datagram_msg(skb, rxm->offset,
2196                                                     msg, chunk);
2197                         if (err < 0)
2198                                 goto put_on_rx_list_err;
2199
2200                         if (is_peek) {
2201                                 peeked += chunk;
2202                                 goto put_on_rx_list;
2203                         }
2204
2205                         if (partially_consumed) {
2206                                 rxm->offset += chunk;
2207                                 rxm->full_len -= chunk;
2208                                 goto put_on_rx_list;
2209                         }
2210
2211                         consume_skb(skb);
2212                 }
2213
2214                 decrypted += chunk;
2215                 len -= chunk;
2216
2217                 /* Return full control message to userspace before trying
2218                  * to parse another message type
2219                  */
2220                 msg->msg_flags |= MSG_EOR;
2221                 if (control != TLS_RECORD_TYPE_DATA)
2222                         break;
2223         }
2224
2225 recv_end:
2226         if (async) {
2227                 int ret;
2228
2229                 /* Wait for all previously submitted records to be decrypted */
2230                 ret = tls_decrypt_async_wait(ctx);
2231                 __skb_queue_purge(&ctx->async_hold);
2232
2233                 if (ret) {
2234                         if (err >= 0 || err == -EINPROGRESS)
2235                                 err = ret;
2236                         decrypted = 0;
2237                         goto end;
2238                 }
2239
2240                 /* Drain records from the rx_list & copy if required */
2241                 if (is_peek)
2242                         err = process_rx_list(ctx, msg, &control, copied + peeked,
2243                                               decrypted - peeked, is_peek, NULL);
2244                 else
2245                         err = process_rx_list(ctx, msg, &control, 0,
2246                                               async_copy_bytes, is_peek, NULL);
2247
2248                 /* we could have copied less than we wanted, and possibly nothing */
2249                 decrypted += max(err, 0) - async_copy_bytes;
2250         }
2251
2252         copied += decrypted;
2253
2254 end:
2255         tls_rx_reader_unlock(sk, ctx);
2256         if (psock)
2257                 sk_psock_put(sk, psock);
2258         return copied ? : err;
2259 }
2260
2261 ssize_t tls_sw_splice_read(struct socket *sock,  loff_t *ppos,
2262                            struct pipe_inode_info *pipe,
2263                            size_t len, unsigned int flags)
2264 {
2265         struct tls_context *tls_ctx = tls_get_ctx(sock->sk);
2266         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2267         struct strp_msg *rxm = NULL;
2268         struct sock *sk = sock->sk;
2269         struct tls_msg *tlm;
2270         struct sk_buff *skb;
2271         ssize_t copied = 0;
2272         int chunk;
2273         int err;
2274
2275         err = tls_rx_reader_lock(sk, ctx, flags & SPLICE_F_NONBLOCK);
2276         if (err < 0)
2277                 return err;
2278
2279         if (!skb_queue_empty(&ctx->rx_list)) {
2280                 skb = __skb_dequeue(&ctx->rx_list);
2281         } else {
2282                 struct tls_decrypt_arg darg;
2283
2284                 err = tls_rx_rec_wait(sk, NULL, flags & SPLICE_F_NONBLOCK,
2285                                       true);
2286                 if (err <= 0)
2287                         goto splice_read_end;
2288
2289                 memset(&darg.inargs, 0, sizeof(darg.inargs));
2290
2291                 err = tls_rx_one_record(sk, NULL, &darg);
2292                 if (err < 0) {
2293                         tls_err_abort(sk, -EBADMSG);
2294                         goto splice_read_end;
2295                 }
2296
2297                 tls_rx_rec_done(ctx);
2298                 skb = darg.skb;
2299         }
2300
2301         rxm = strp_msg(skb);
2302         tlm = tls_msg(skb);
2303
2304         /* splice does not support reading control messages */
2305         if (tlm->control != TLS_RECORD_TYPE_DATA) {
2306                 err = -EINVAL;
2307                 goto splice_requeue;
2308         }
2309
2310         chunk = min_t(unsigned int, rxm->full_len, len);
2311         copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags);
2312         if (copied < 0)
2313                 goto splice_requeue;
2314
2315         if (chunk < rxm->full_len) {
2316                 rxm->offset += len;
2317                 rxm->full_len -= len;
2318                 goto splice_requeue;
2319         }
2320
2321         consume_skb(skb);
2322
2323 splice_read_end:
2324         tls_rx_reader_unlock(sk, ctx);
2325         return copied ? : err;
2326
2327 splice_requeue:
2328         __skb_queue_head(&ctx->rx_list, skb);
2329         goto splice_read_end;
2330 }
2331
2332 bool tls_sw_sock_is_readable(struct sock *sk)
2333 {
2334         struct tls_context *tls_ctx = tls_get_ctx(sk);
2335         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2336         bool ingress_empty = true;
2337         struct sk_psock *psock;
2338
2339         rcu_read_lock();
2340         psock = sk_psock(sk);
2341         if (psock)
2342                 ingress_empty = list_empty(&psock->ingress_msg);
2343         rcu_read_unlock();
2344
2345         return !ingress_empty || tls_strp_msg_ready(ctx) ||
2346                 !skb_queue_empty(&ctx->rx_list);
2347 }
2348
2349 int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb)
2350 {
2351         struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
2352         struct tls_prot_info *prot = &tls_ctx->prot_info;
2353         char header[TLS_HEADER_SIZE + MAX_IV_SIZE];
2354         size_t cipher_overhead;
2355         size_t data_len = 0;
2356         int ret;
2357
2358         /* Verify that we have a full TLS header, or wait for more data */
2359         if (strp->stm.offset + prot->prepend_size > skb->len)
2360                 return 0;
2361
2362         /* Sanity-check size of on-stack buffer. */
2363         if (WARN_ON(prot->prepend_size > sizeof(header))) {
2364                 ret = -EINVAL;
2365                 goto read_failure;
2366         }
2367
2368         /* Linearize header to local buffer */
2369         ret = skb_copy_bits(skb, strp->stm.offset, header, prot->prepend_size);
2370         if (ret < 0)
2371                 goto read_failure;
2372
2373         strp->mark = header[0];
2374
2375         data_len = ((header[4] & 0xFF) | (header[3] << 8));
2376
2377         cipher_overhead = prot->tag_size;
2378         if (prot->version != TLS_1_3_VERSION &&
2379             prot->cipher_type != TLS_CIPHER_CHACHA20_POLY1305)
2380                 cipher_overhead += prot->iv_size;
2381
2382         if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead +
2383             prot->tail_size) {
2384                 ret = -EMSGSIZE;
2385                 goto read_failure;
2386         }
2387         if (data_len < cipher_overhead) {
2388                 ret = -EBADMSG;
2389                 goto read_failure;
2390         }
2391
2392         /* Note that both TLS1.3 and TLS1.2 use TLS_1_2 version here */
2393         if (header[1] != TLS_1_2_VERSION_MINOR ||
2394             header[2] != TLS_1_2_VERSION_MAJOR) {
2395                 ret = -EINVAL;
2396                 goto read_failure;
2397         }
2398
2399         tls_device_rx_resync_new_rec(strp->sk, data_len + TLS_HEADER_SIZE,
2400                                      TCP_SKB_CB(skb)->seq + strp->stm.offset);
2401         return data_len + TLS_HEADER_SIZE;
2402
2403 read_failure:
2404         tls_err_abort(strp->sk, ret);
2405
2406         return ret;
2407 }
2408
2409 void tls_rx_msg_ready(struct tls_strparser *strp)
2410 {
2411         struct tls_sw_context_rx *ctx;
2412
2413         ctx = container_of(strp, struct tls_sw_context_rx, strp);
2414         ctx->saved_data_ready(strp->sk);
2415 }
2416
2417 static void tls_data_ready(struct sock *sk)
2418 {
2419         struct tls_context *tls_ctx = tls_get_ctx(sk);
2420         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2421         struct sk_psock *psock;
2422         gfp_t alloc_save;
2423
2424         alloc_save = sk->sk_allocation;
2425         sk->sk_allocation = GFP_ATOMIC;
2426         tls_strp_data_ready(&ctx->strp);
2427         sk->sk_allocation = alloc_save;
2428
2429         psock = sk_psock_get(sk);
2430         if (psock) {
2431                 if (!list_empty(&psock->ingress_msg))
2432                         ctx->saved_data_ready(sk);
2433                 sk_psock_put(sk, psock);
2434         }
2435 }
2436
2437 void tls_sw_cancel_work_tx(struct tls_context *tls_ctx)
2438 {
2439         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
2440
2441         set_bit(BIT_TX_CLOSING, &ctx->tx_bitmask);
2442         set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask);
2443         cancel_delayed_work_sync(&ctx->tx_work.work);
2444 }
2445
2446 void tls_sw_release_resources_tx(struct sock *sk)
2447 {
2448         struct tls_context *tls_ctx = tls_get_ctx(sk);
2449         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
2450         struct tls_rec *rec, *tmp;
2451
2452         /* Wait for any pending async encryptions to complete */
2453         tls_encrypt_async_wait(ctx);
2454
2455         tls_tx_records(sk, -1);
2456
2457         /* Free up un-sent records in tx_list. First, free
2458          * the partially sent record if any at head of tx_list.
2459          */
2460         if (tls_ctx->partially_sent_record) {
2461                 tls_free_partial_record(sk, tls_ctx);
2462                 rec = list_first_entry(&ctx->tx_list,
2463                                        struct tls_rec, list);
2464                 list_del(&rec->list);
2465                 sk_msg_free(sk, &rec->msg_plaintext);
2466                 kfree(rec);
2467         }
2468
2469         list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) {
2470                 list_del(&rec->list);
2471                 sk_msg_free(sk, &rec->msg_encrypted);
2472                 sk_msg_free(sk, &rec->msg_plaintext);
2473                 kfree(rec);
2474         }
2475
2476         crypto_free_aead(ctx->aead_send);
2477         tls_free_open_rec(sk);
2478 }
2479
2480 void tls_sw_free_ctx_tx(struct tls_context *tls_ctx)
2481 {
2482         struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
2483
2484         kfree(ctx);
2485 }
2486
2487 void tls_sw_release_resources_rx(struct sock *sk)
2488 {
2489         struct tls_context *tls_ctx = tls_get_ctx(sk);
2490         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2491
2492         kfree(tls_ctx->rx.rec_seq);
2493         kfree(tls_ctx->rx.iv);
2494
2495         if (ctx->aead_recv) {
2496                 __skb_queue_purge(&ctx->rx_list);
2497                 crypto_free_aead(ctx->aead_recv);
2498                 tls_strp_stop(&ctx->strp);
2499                 /* If tls_sw_strparser_arm() was not called (cleanup paths)
2500                  * we still want to tls_strp_stop(), but sk->sk_data_ready was
2501                  * never swapped.
2502                  */
2503                 if (ctx->saved_data_ready) {
2504                         write_lock_bh(&sk->sk_callback_lock);
2505                         sk->sk_data_ready = ctx->saved_data_ready;
2506                         write_unlock_bh(&sk->sk_callback_lock);
2507                 }
2508         }
2509 }
2510
2511 void tls_sw_strparser_done(struct tls_context *tls_ctx)
2512 {
2513         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2514
2515         tls_strp_done(&ctx->strp);
2516 }
2517
2518 void tls_sw_free_ctx_rx(struct tls_context *tls_ctx)
2519 {
2520         struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2521
2522         kfree(ctx);
2523 }
2524
2525 void tls_sw_free_resources_rx(struct sock *sk)
2526 {
2527         struct tls_context *tls_ctx = tls_get_ctx(sk);
2528
2529         tls_sw_release_resources_rx(sk);
2530         tls_sw_free_ctx_rx(tls_ctx);
2531 }
2532
2533 /* The work handler to transmitt the encrypted records in tx_list */
2534 static void tx_work_handler(struct work_struct *work)
2535 {
2536         struct delayed_work *delayed_work = to_delayed_work(work);
2537         struct tx_work *tx_work = container_of(delayed_work,
2538                                                struct tx_work, work);
2539         struct sock *sk = tx_work->sk;
2540         struct tls_context *tls_ctx = tls_get_ctx(sk);
2541         struct tls_sw_context_tx *ctx;
2542
2543         if (unlikely(!tls_ctx))
2544                 return;
2545
2546         ctx = tls_sw_ctx_tx(tls_ctx);
2547         if (test_bit(BIT_TX_CLOSING, &ctx->tx_bitmask))
2548                 return;
2549
2550         if (!test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask))
2551                 return;
2552
2553         if (mutex_trylock(&tls_ctx->tx_lock)) {
2554                 lock_sock(sk);
2555                 tls_tx_records(sk, -1);
2556                 release_sock(sk);
2557                 mutex_unlock(&tls_ctx->tx_lock);
2558         } else if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
2559                 /* Someone is holding the tx_lock, they will likely run Tx
2560                  * and cancel the work on their way out of the lock section.
2561                  * Schedule a long delay just in case.
2562                  */
2563                 schedule_delayed_work(&ctx->tx_work.work, msecs_to_jiffies(10));
2564         }
2565 }
2566
2567 static bool tls_is_tx_ready(struct tls_sw_context_tx *ctx)
2568 {
2569         struct tls_rec *rec;
2570
2571         rec = list_first_entry_or_null(&ctx->tx_list, struct tls_rec, list);
2572         if (!rec)
2573                 return false;
2574
2575         return READ_ONCE(rec->tx_ready);
2576 }
2577
2578 void tls_sw_write_space(struct sock *sk, struct tls_context *ctx)
2579 {
2580         struct tls_sw_context_tx *tx_ctx = tls_sw_ctx_tx(ctx);
2581
2582         /* Schedule the transmission if tx list is ready */
2583         if (tls_is_tx_ready(tx_ctx) &&
2584             !test_and_set_bit(BIT_TX_SCHEDULED, &tx_ctx->tx_bitmask))
2585                 schedule_delayed_work(&tx_ctx->tx_work.work, 0);
2586 }
2587
2588 void tls_sw_strparser_arm(struct sock *sk, struct tls_context *tls_ctx)
2589 {
2590         struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(tls_ctx);
2591
2592         write_lock_bh(&sk->sk_callback_lock);
2593         rx_ctx->saved_data_ready = sk->sk_data_ready;
2594         sk->sk_data_ready = tls_data_ready;
2595         write_unlock_bh(&sk->sk_callback_lock);
2596 }
2597
2598 void tls_update_rx_zc_capable(struct tls_context *tls_ctx)
2599 {
2600         struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(tls_ctx);
2601
2602         rx_ctx->zc_capable = tls_ctx->rx_no_pad ||
2603                 tls_ctx->prot_info.version != TLS_1_3_VERSION;
2604 }
2605
2606 static struct tls_sw_context_tx *init_ctx_tx(struct tls_context *ctx, struct sock *sk)
2607 {
2608         struct tls_sw_context_tx *sw_ctx_tx;
2609
2610         if (!ctx->priv_ctx_tx) {
2611                 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL);
2612                 if (!sw_ctx_tx)
2613                         return NULL;
2614         } else {
2615                 sw_ctx_tx = ctx->priv_ctx_tx;
2616         }
2617
2618         crypto_init_wait(&sw_ctx_tx->async_wait);
2619         atomic_set(&sw_ctx_tx->encrypt_pending, 1);
2620         INIT_LIST_HEAD(&sw_ctx_tx->tx_list);
2621         INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler);
2622         sw_ctx_tx->tx_work.sk = sk;
2623
2624         return sw_ctx_tx;
2625 }
2626
2627 static struct tls_sw_context_rx *init_ctx_rx(struct tls_context *ctx)
2628 {
2629         struct tls_sw_context_rx *sw_ctx_rx;
2630
2631         if (!ctx->priv_ctx_rx) {
2632                 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL);
2633                 if (!sw_ctx_rx)
2634                         return NULL;
2635         } else {
2636                 sw_ctx_rx = ctx->priv_ctx_rx;
2637         }
2638
2639         crypto_init_wait(&sw_ctx_rx->async_wait);
2640         atomic_set(&sw_ctx_rx->decrypt_pending, 1);
2641         init_waitqueue_head(&sw_ctx_rx->wq);
2642         skb_queue_head_init(&sw_ctx_rx->rx_list);
2643         skb_queue_head_init(&sw_ctx_rx->async_hold);
2644
2645         return sw_ctx_rx;
2646 }
2647
2648 int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
2649 {
2650         struct tls_context *tls_ctx = tls_get_ctx(sk);
2651         struct tls_prot_info *prot = &tls_ctx->prot_info;
2652         struct tls_crypto_info *crypto_info;
2653         struct tls_sw_context_tx *sw_ctx_tx = NULL;
2654         struct tls_sw_context_rx *sw_ctx_rx = NULL;
2655         struct cipher_context *cctx;
2656         struct crypto_aead **aead;
2657         u16 nonce_size, tag_size, iv_size, rec_seq_size, salt_size;
2658         struct crypto_tfm *tfm;
2659         char *iv, *rec_seq, *key, *salt, *cipher_name;
2660         size_t keysize;
2661         int rc = 0;
2662
2663         if (!ctx) {
2664                 rc = -EINVAL;
2665                 goto out;
2666         }
2667
2668         if (tx) {
2669                 ctx->priv_ctx_tx = init_ctx_tx(ctx, sk);
2670                 if (!ctx->priv_ctx_tx)
2671                         return -ENOMEM;
2672
2673                 sw_ctx_tx = ctx->priv_ctx_tx;
2674                 crypto_info = &ctx->crypto_send.info;
2675                 cctx = &ctx->tx;
2676                 aead = &sw_ctx_tx->aead_send;
2677         } else {
2678                 ctx->priv_ctx_rx = init_ctx_rx(ctx);
2679                 if (!ctx->priv_ctx_rx)
2680                         return -ENOMEM;
2681
2682                 sw_ctx_rx = ctx->priv_ctx_rx;
2683                 crypto_info = &ctx->crypto_recv.info;
2684                 cctx = &ctx->rx;
2685                 aead = &sw_ctx_rx->aead_recv;
2686         }
2687
2688         switch (crypto_info->cipher_type) {
2689         case TLS_CIPHER_AES_GCM_128: {
2690                 struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
2691
2692                 gcm_128_info = (void *)crypto_info;
2693                 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
2694                 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
2695                 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
2696                 iv = gcm_128_info->iv;
2697                 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
2698                 rec_seq = gcm_128_info->rec_seq;
2699                 keysize = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
2700                 key = gcm_128_info->key;
2701                 salt = gcm_128_info->salt;
2702                 salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
2703                 cipher_name = "gcm(aes)";
2704                 break;
2705         }
2706         case TLS_CIPHER_AES_GCM_256: {
2707                 struct tls12_crypto_info_aes_gcm_256 *gcm_256_info;
2708
2709                 gcm_256_info = (void *)crypto_info;
2710                 nonce_size = TLS_CIPHER_AES_GCM_256_IV_SIZE;
2711                 tag_size = TLS_CIPHER_AES_GCM_256_TAG_SIZE;
2712                 iv_size = TLS_CIPHER_AES_GCM_256_IV_SIZE;
2713                 iv = gcm_256_info->iv;
2714                 rec_seq_size = TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE;
2715                 rec_seq = gcm_256_info->rec_seq;
2716                 keysize = TLS_CIPHER_AES_GCM_256_KEY_SIZE;
2717                 key = gcm_256_info->key;
2718                 salt = gcm_256_info->salt;
2719                 salt_size = TLS_CIPHER_AES_GCM_256_SALT_SIZE;
2720                 cipher_name = "gcm(aes)";
2721                 break;
2722         }
2723         case TLS_CIPHER_AES_CCM_128: {
2724                 struct tls12_crypto_info_aes_ccm_128 *ccm_128_info;
2725
2726                 ccm_128_info = (void *)crypto_info;
2727                 nonce_size = TLS_CIPHER_AES_CCM_128_IV_SIZE;
2728                 tag_size = TLS_CIPHER_AES_CCM_128_TAG_SIZE;
2729                 iv_size = TLS_CIPHER_AES_CCM_128_IV_SIZE;
2730                 iv = ccm_128_info->iv;
2731                 rec_seq_size = TLS_CIPHER_AES_CCM_128_REC_SEQ_SIZE;
2732                 rec_seq = ccm_128_info->rec_seq;
2733                 keysize = TLS_CIPHER_AES_CCM_128_KEY_SIZE;
2734                 key = ccm_128_info->key;
2735                 salt = ccm_128_info->salt;
2736                 salt_size = TLS_CIPHER_AES_CCM_128_SALT_SIZE;
2737                 cipher_name = "ccm(aes)";
2738                 break;
2739         }
2740         case TLS_CIPHER_CHACHA20_POLY1305: {
2741                 struct tls12_crypto_info_chacha20_poly1305 *chacha20_poly1305_info;
2742
2743                 chacha20_poly1305_info = (void *)crypto_info;
2744                 nonce_size = 0;
2745                 tag_size = TLS_CIPHER_CHACHA20_POLY1305_TAG_SIZE;
2746                 iv_size = TLS_CIPHER_CHACHA20_POLY1305_IV_SIZE;
2747                 iv = chacha20_poly1305_info->iv;
2748                 rec_seq_size = TLS_CIPHER_CHACHA20_POLY1305_REC_SEQ_SIZE;
2749                 rec_seq = chacha20_poly1305_info->rec_seq;
2750                 keysize = TLS_CIPHER_CHACHA20_POLY1305_KEY_SIZE;
2751                 key = chacha20_poly1305_info->key;
2752                 salt = chacha20_poly1305_info->salt;
2753                 salt_size = TLS_CIPHER_CHACHA20_POLY1305_SALT_SIZE;
2754                 cipher_name = "rfc7539(chacha20,poly1305)";
2755                 break;
2756         }
2757         case TLS_CIPHER_SM4_GCM: {
2758                 struct tls12_crypto_info_sm4_gcm *sm4_gcm_info;
2759
2760                 sm4_gcm_info = (void *)crypto_info;
2761                 nonce_size = TLS_CIPHER_SM4_GCM_IV_SIZE;
2762                 tag_size = TLS_CIPHER_SM4_GCM_TAG_SIZE;
2763                 iv_size = TLS_CIPHER_SM4_GCM_IV_SIZE;
2764                 iv = sm4_gcm_info->iv;
2765                 rec_seq_size = TLS_CIPHER_SM4_GCM_REC_SEQ_SIZE;
2766                 rec_seq = sm4_gcm_info->rec_seq;
2767                 keysize = TLS_CIPHER_SM4_GCM_KEY_SIZE;
2768                 key = sm4_gcm_info->key;
2769                 salt = sm4_gcm_info->salt;
2770                 salt_size = TLS_CIPHER_SM4_GCM_SALT_SIZE;
2771                 cipher_name = "gcm(sm4)";
2772                 break;
2773         }
2774         case TLS_CIPHER_SM4_CCM: {
2775                 struct tls12_crypto_info_sm4_ccm *sm4_ccm_info;
2776
2777                 sm4_ccm_info = (void *)crypto_info;
2778                 nonce_size = TLS_CIPHER_SM4_CCM_IV_SIZE;
2779                 tag_size = TLS_CIPHER_SM4_CCM_TAG_SIZE;
2780                 iv_size = TLS_CIPHER_SM4_CCM_IV_SIZE;
2781                 iv = sm4_ccm_info->iv;
2782                 rec_seq_size = TLS_CIPHER_SM4_CCM_REC_SEQ_SIZE;
2783                 rec_seq = sm4_ccm_info->rec_seq;
2784                 keysize = TLS_CIPHER_SM4_CCM_KEY_SIZE;
2785                 key = sm4_ccm_info->key;
2786                 salt = sm4_ccm_info->salt;
2787                 salt_size = TLS_CIPHER_SM4_CCM_SALT_SIZE;
2788                 cipher_name = "ccm(sm4)";
2789                 break;
2790         }
2791         case TLS_CIPHER_ARIA_GCM_128: {
2792                 struct tls12_crypto_info_aria_gcm_128 *aria_gcm_128_info;
2793
2794                 aria_gcm_128_info = (void *)crypto_info;
2795                 nonce_size = TLS_CIPHER_ARIA_GCM_128_IV_SIZE;
2796                 tag_size = TLS_CIPHER_ARIA_GCM_128_TAG_SIZE;
2797                 iv_size = TLS_CIPHER_ARIA_GCM_128_IV_SIZE;
2798                 iv = aria_gcm_128_info->iv;
2799                 rec_seq_size = TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE;
2800                 rec_seq = aria_gcm_128_info->rec_seq;
2801                 keysize = TLS_CIPHER_ARIA_GCM_128_KEY_SIZE;
2802                 key = aria_gcm_128_info->key;
2803                 salt = aria_gcm_128_info->salt;
2804                 salt_size = TLS_CIPHER_ARIA_GCM_128_SALT_SIZE;
2805                 cipher_name = "gcm(aria)";
2806                 break;
2807         }
2808         case TLS_CIPHER_ARIA_GCM_256: {
2809                 struct tls12_crypto_info_aria_gcm_256 *gcm_256_info;
2810
2811                 gcm_256_info = (void *)crypto_info;
2812                 nonce_size = TLS_CIPHER_ARIA_GCM_256_IV_SIZE;
2813                 tag_size = TLS_CIPHER_ARIA_GCM_256_TAG_SIZE;
2814                 iv_size = TLS_CIPHER_ARIA_GCM_256_IV_SIZE;
2815                 iv = gcm_256_info->iv;
2816                 rec_seq_size = TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE;
2817                 rec_seq = gcm_256_info->rec_seq;
2818                 keysize = TLS_CIPHER_ARIA_GCM_256_KEY_SIZE;
2819                 key = gcm_256_info->key;
2820                 salt = gcm_256_info->salt;
2821                 salt_size = TLS_CIPHER_ARIA_GCM_256_SALT_SIZE;
2822                 cipher_name = "gcm(aria)";
2823                 break;
2824         }
2825         default:
2826                 rc = -EINVAL;
2827                 goto free_priv;
2828         }
2829
2830         if (crypto_info->version == TLS_1_3_VERSION) {
2831                 nonce_size = 0;
2832                 prot->aad_size = TLS_HEADER_SIZE;
2833                 prot->tail_size = 1;
2834         } else {
2835                 prot->aad_size = TLS_AAD_SPACE_SIZE;
2836                 prot->tail_size = 0;
2837         }
2838
2839         /* Sanity-check the sizes for stack allocations. */
2840         if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE ||
2841             rec_seq_size > TLS_MAX_REC_SEQ_SIZE || tag_size != TLS_TAG_SIZE ||
2842             prot->aad_size > TLS_MAX_AAD_SIZE) {
2843                 rc = -EINVAL;
2844                 goto free_priv;
2845         }
2846
2847         prot->version = crypto_info->version;
2848         prot->cipher_type = crypto_info->cipher_type;
2849         prot->prepend_size = TLS_HEADER_SIZE + nonce_size;
2850         prot->tag_size = tag_size;
2851         prot->overhead_size = prot->prepend_size +
2852                               prot->tag_size + prot->tail_size;
2853         prot->iv_size = iv_size;
2854         prot->salt_size = salt_size;
2855         cctx->iv = kmalloc(iv_size + salt_size, GFP_KERNEL);
2856         if (!cctx->iv) {
2857                 rc = -ENOMEM;
2858                 goto free_priv;
2859         }
2860         /* Note: 128 & 256 bit salt are the same size */
2861         prot->rec_seq_size = rec_seq_size;
2862         memcpy(cctx->iv, salt, salt_size);
2863         memcpy(cctx->iv + salt_size, iv, iv_size);
2864         cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL);
2865         if (!cctx->rec_seq) {
2866                 rc = -ENOMEM;
2867                 goto free_iv;
2868         }
2869
2870         if (!*aead) {
2871                 *aead = crypto_alloc_aead(cipher_name, 0, 0);
2872                 if (IS_ERR(*aead)) {
2873                         rc = PTR_ERR(*aead);
2874                         *aead = NULL;
2875                         goto free_rec_seq;
2876                 }
2877         }
2878
2879         ctx->push_pending_record = tls_sw_push_pending_record;
2880
2881         rc = crypto_aead_setkey(*aead, key, keysize);
2882
2883         if (rc)
2884                 goto free_aead;
2885
2886         rc = crypto_aead_setauthsize(*aead, prot->tag_size);
2887         if (rc)
2888                 goto free_aead;
2889
2890         if (sw_ctx_rx) {
2891                 tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv);
2892
2893                 tls_update_rx_zc_capable(ctx);
2894                 sw_ctx_rx->async_capable =
2895                         crypto_info->version != TLS_1_3_VERSION &&
2896                         !!(tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC);
2897
2898                 rc = tls_strp_init(&sw_ctx_rx->strp, sk);
2899                 if (rc)
2900                         goto free_aead;
2901         }
2902
2903         goto out;
2904
2905 free_aead:
2906         crypto_free_aead(*aead);
2907         *aead = NULL;
2908 free_rec_seq:
2909         kfree(cctx->rec_seq);
2910         cctx->rec_seq = NULL;
2911 free_iv:
2912         kfree(cctx->iv);
2913         cctx->iv = NULL;
2914 free_priv:
2915         if (tx) {
2916                 kfree(ctx->priv_ctx_tx);
2917                 ctx->priv_ctx_tx = NULL;
2918         } else {
2919                 kfree(ctx->priv_ctx_rx);
2920                 ctx->priv_ctx_rx = NULL;
2921         }
2922 out:
2923         return rc;
2924 }