2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4 * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
5 * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
6 * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37 #include <linux/sched/signal.h>
38 #include <linux/module.h>
39 #include <crypto/aead.h>
41 #include <net/strparser.h>
44 #define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE
46 static int tls_do_decryption(struct sock *sk,
47 struct scatterlist *sgin,
48 struct scatterlist *sgout,
51 struct aead_request *aead_req)
53 struct tls_context *tls_ctx = tls_get_ctx(sk);
54 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
57 aead_request_set_tfm(aead_req, ctx->aead_recv);
58 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
59 aead_request_set_crypt(aead_req, sgin, sgout,
60 data_len + tls_ctx->rx.tag_size,
62 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
63 crypto_req_done, &ctx->async_wait);
65 ret = crypto_wait_req(crypto_aead_decrypt(aead_req), &ctx->async_wait);
69 static void trim_sg(struct sock *sk, struct scatterlist *sg,
70 int *sg_num_elem, unsigned int *sg_size, int target_size)
72 int i = *sg_num_elem - 1;
73 int trim = *sg_size - target_size;
80 *sg_size = target_size;
81 while (trim >= sg[i].length) {
83 sk_mem_uncharge(sk, sg[i].length);
84 put_page(sg_page(&sg[i]));
92 sk_mem_uncharge(sk, trim);
98 static void trim_both_sgl(struct sock *sk, int target_size)
100 struct tls_context *tls_ctx = tls_get_ctx(sk);
101 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
103 trim_sg(sk, ctx->sg_plaintext_data,
104 &ctx->sg_plaintext_num_elem,
105 &ctx->sg_plaintext_size,
109 target_size += tls_ctx->tx.overhead_size;
111 trim_sg(sk, ctx->sg_encrypted_data,
112 &ctx->sg_encrypted_num_elem,
113 &ctx->sg_encrypted_size,
117 static int alloc_encrypted_sg(struct sock *sk, int len)
119 struct tls_context *tls_ctx = tls_get_ctx(sk);
120 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
123 rc = sk_alloc_sg(sk, len,
124 ctx->sg_encrypted_data, 0,
125 &ctx->sg_encrypted_num_elem,
126 &ctx->sg_encrypted_size, 0);
129 ctx->sg_encrypted_num_elem = ARRAY_SIZE(ctx->sg_encrypted_data);
134 static int alloc_plaintext_sg(struct sock *sk, int len)
136 struct tls_context *tls_ctx = tls_get_ctx(sk);
137 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
140 rc = sk_alloc_sg(sk, len, ctx->sg_plaintext_data, 0,
141 &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size,
142 tls_ctx->pending_open_record_frags);
145 ctx->sg_plaintext_num_elem = ARRAY_SIZE(ctx->sg_plaintext_data);
150 static void free_sg(struct sock *sk, struct scatterlist *sg,
151 int *sg_num_elem, unsigned int *sg_size)
153 int i, n = *sg_num_elem;
155 for (i = 0; i < n; ++i) {
156 sk_mem_uncharge(sk, sg[i].length);
157 put_page(sg_page(&sg[i]));
163 static void tls_free_both_sg(struct sock *sk)
165 struct tls_context *tls_ctx = tls_get_ctx(sk);
166 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
168 free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem,
169 &ctx->sg_encrypted_size);
171 free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
172 &ctx->sg_plaintext_size);
175 static int tls_do_encryption(struct tls_context *tls_ctx,
176 struct tls_sw_context_tx *ctx,
177 struct aead_request *aead_req,
182 ctx->sg_encrypted_data[0].offset += tls_ctx->tx.prepend_size;
183 ctx->sg_encrypted_data[0].length -= tls_ctx->tx.prepend_size;
185 aead_request_set_tfm(aead_req, ctx->aead_send);
186 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
187 aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out,
188 data_len, tls_ctx->tx.iv);
190 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
191 crypto_req_done, &ctx->async_wait);
193 rc = crypto_wait_req(crypto_aead_encrypt(aead_req), &ctx->async_wait);
195 ctx->sg_encrypted_data[0].offset -= tls_ctx->tx.prepend_size;
196 ctx->sg_encrypted_data[0].length += tls_ctx->tx.prepend_size;
201 static int tls_push_record(struct sock *sk, int flags,
202 unsigned char record_type)
204 struct tls_context *tls_ctx = tls_get_ctx(sk);
205 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
206 struct aead_request *req;
209 req = aead_request_alloc(ctx->aead_send, sk->sk_allocation);
213 sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1);
214 sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1);
216 tls_make_aad(ctx->aad_space, ctx->sg_plaintext_size,
217 tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size,
220 tls_fill_prepend(tls_ctx,
221 page_address(sg_page(&ctx->sg_encrypted_data[0])) +
222 ctx->sg_encrypted_data[0].offset,
223 ctx->sg_plaintext_size, record_type);
225 tls_ctx->pending_open_record_frags = 0;
226 set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags);
228 rc = tls_do_encryption(tls_ctx, ctx, req, ctx->sg_plaintext_size);
230 /* If we are called from write_space and
231 * we fail, we need to set this SOCK_NOSPACE
232 * to trigger another write_space in the future.
234 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
238 free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
239 &ctx->sg_plaintext_size);
241 ctx->sg_encrypted_num_elem = 0;
242 ctx->sg_encrypted_size = 0;
244 /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */
245 rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags);
246 if (rc < 0 && rc != -EAGAIN)
247 tls_err_abort(sk, EBADMSG);
249 tls_advance_record_sn(sk, &tls_ctx->tx);
251 aead_request_free(req);
255 static int tls_sw_push_pending_record(struct sock *sk, int flags)
257 return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA);
260 static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
261 int length, int *pages_used,
262 unsigned int *size_used,
263 struct scatterlist *to, int to_max_pages,
266 struct page *pages[MAX_SKB_FRAGS];
271 unsigned int size = *size_used;
272 int num_elem = *pages_used;
278 maxpages = to_max_pages - num_elem;
283 copied = iov_iter_get_pages(from, pages,
291 iov_iter_advance(from, copied);
296 use = min_t(int, copied, PAGE_SIZE - offset);
298 sg_set_page(&to[num_elem],
299 pages[i], use, offset);
300 sg_unmark_end(&to[num_elem]);
302 sk_mem_charge(sk, use);
312 /* Mark the end in the last sg entry if newly added */
313 if (num_elem > *pages_used)
314 sg_mark_end(&to[num_elem - 1]);
317 iov_iter_revert(from, size - *size_used);
319 *pages_used = num_elem;
324 static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
327 struct tls_context *tls_ctx = tls_get_ctx(sk);
328 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
329 struct scatterlist *sg = ctx->sg_plaintext_data;
332 for (i = tls_ctx->pending_open_record_frags;
333 i < ctx->sg_plaintext_num_elem; ++i) {
336 page_address(sg_page(&sg[i])) + sg[i].offset,
337 copy, from) != copy) {
343 ++tls_ctx->pending_open_record_frags;
353 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
355 struct tls_context *tls_ctx = tls_get_ctx(sk);
356 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
359 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
360 bool eor = !(msg->msg_flags & MSG_MORE);
361 size_t try_to_copy, copied = 0;
362 unsigned char record_type = TLS_RECORD_TYPE_DATA;
366 bool is_kvec = msg->msg_iter.type & ITER_KVEC;
368 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
373 ret = tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo);
377 if (unlikely(msg->msg_controllen)) {
378 ret = tls_proccess_cmsg(sk, msg, &record_type);
383 while (msg_data_left(msg)) {
389 orig_size = ctx->sg_plaintext_size;
391 try_to_copy = msg_data_left(msg);
392 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
393 if (try_to_copy >= record_room) {
394 try_to_copy = record_room;
398 required_size = ctx->sg_plaintext_size + try_to_copy +
399 tls_ctx->tx.overhead_size;
401 if (!sk_stream_memory_free(sk))
402 goto wait_for_sndbuf;
404 ret = alloc_encrypted_sg(sk, required_size);
407 goto wait_for_memory;
409 /* Adjust try_to_copy according to the amount that was
410 * actually allocated. The difference is due
411 * to max sg elements limit
413 try_to_copy -= required_size - ctx->sg_encrypted_size;
416 if (!is_kvec && (full_record || eor)) {
417 ret = zerocopy_from_iter(sk, &msg->msg_iter,
418 try_to_copy, &ctx->sg_plaintext_num_elem,
419 &ctx->sg_plaintext_size,
420 ctx->sg_plaintext_data,
421 ARRAY_SIZE(ctx->sg_plaintext_data),
424 goto fallback_to_reg_send;
426 copied += try_to_copy;
427 ret = tls_push_record(sk, msg->msg_flags, record_type);
432 fallback_to_reg_send:
433 trim_sg(sk, ctx->sg_plaintext_data,
434 &ctx->sg_plaintext_num_elem,
435 &ctx->sg_plaintext_size,
439 required_size = ctx->sg_plaintext_size + try_to_copy;
441 ret = alloc_plaintext_sg(sk, required_size);
444 goto wait_for_memory;
446 /* Adjust try_to_copy according to the amount that was
447 * actually allocated. The difference is due
448 * to max sg elements limit
450 try_to_copy -= required_size - ctx->sg_plaintext_size;
453 trim_sg(sk, ctx->sg_encrypted_data,
454 &ctx->sg_encrypted_num_elem,
455 &ctx->sg_encrypted_size,
456 ctx->sg_plaintext_size +
457 tls_ctx->tx.overhead_size);
460 ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy);
464 copied += try_to_copy;
465 if (full_record || eor) {
467 ret = tls_push_record(sk, msg->msg_flags, record_type);
470 goto wait_for_memory;
479 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
481 ret = sk_stream_wait_memory(sk, &timeo);
484 trim_both_sgl(sk, orig_size);
488 if (tls_is_pending_closed_record(tls_ctx))
491 if (ctx->sg_encrypted_size < required_size)
492 goto alloc_encrypted;
494 goto alloc_plaintext;
498 ret = sk_stream_error(sk, msg->msg_flags, ret);
501 return copied ? copied : ret;
504 int tls_sw_sendpage(struct sock *sk, struct page *page,
505 int offset, size_t size, int flags)
507 struct tls_context *tls_ctx = tls_get_ctx(sk);
508 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
510 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
512 size_t orig_size = size;
513 unsigned char record_type = TLS_RECORD_TYPE_DATA;
514 struct scatterlist *sg;
518 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
519 MSG_SENDPAGE_NOTLAST))
522 /* No MSG_EOR from splice, only look at MSG_MORE */
523 eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
527 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
529 ret = tls_complete_pending_work(sk, tls_ctx, flags, &timeo);
533 /* Call the sk_stream functions to manage the sndbuf mem. */
535 size_t copy, required_size;
543 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
545 if (copy >= record_room) {
549 required_size = ctx->sg_plaintext_size + copy +
550 tls_ctx->tx.overhead_size;
552 if (!sk_stream_memory_free(sk))
553 goto wait_for_sndbuf;
555 ret = alloc_encrypted_sg(sk, required_size);
558 goto wait_for_memory;
560 /* Adjust copy according to the amount that was
561 * actually allocated. The difference is due
562 * to max sg elements limit
564 copy -= required_size - ctx->sg_plaintext_size;
569 sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem;
570 sg_set_page(sg, page, copy, offset);
573 ctx->sg_plaintext_num_elem++;
575 sk_mem_charge(sk, copy);
578 ctx->sg_plaintext_size += copy;
579 tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem;
581 if (full_record || eor ||
582 ctx->sg_plaintext_num_elem ==
583 ARRAY_SIZE(ctx->sg_plaintext_data)) {
585 ret = tls_push_record(sk, flags, record_type);
588 goto wait_for_memory;
595 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
597 ret = sk_stream_wait_memory(sk, &timeo);
599 trim_both_sgl(sk, ctx->sg_plaintext_size);
603 if (tls_is_pending_closed_record(tls_ctx))
610 if (orig_size > size)
611 ret = orig_size - size;
613 ret = sk_stream_error(sk, flags, ret);
619 static struct sk_buff *tls_wait_data(struct sock *sk, int flags,
620 long timeo, int *err)
622 struct tls_context *tls_ctx = tls_get_ctx(sk);
623 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
625 DEFINE_WAIT_FUNC(wait, woken_wake_function);
627 while (!(skb = ctx->recv_pkt)) {
629 *err = sock_error(sk);
633 if (!skb_queue_empty(&sk->sk_receive_queue)) {
634 __strp_unpause(&ctx->strp);
636 return ctx->recv_pkt;
639 if (sk->sk_shutdown & RCV_SHUTDOWN)
642 if (sock_flag(sk, SOCK_DONE))
645 if ((flags & MSG_DONTWAIT) || !timeo) {
650 add_wait_queue(sk_sleep(sk), &wait);
651 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
652 sk_wait_event(sk, &timeo, ctx->recv_pkt != skb, &wait);
653 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
654 remove_wait_queue(sk_sleep(sk), &wait);
657 if (signal_pending(current)) {
658 *err = sock_intr_errno(timeo);
666 /* This function decrypts the input skb into either out_iov or in out_sg
667 * or in skb buffers itself. The input parameter 'zc' indicates if
668 * zero-copy mode needs to be tried or not. With zero-copy mode, either
669 * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are
670 * NULL, then the decryption happens inside skb buffers itself, i.e.
671 * zero-copy gets disabled and 'zc' is updated.
674 static int decrypt_internal(struct sock *sk, struct sk_buff *skb,
675 struct iov_iter *out_iov,
676 struct scatterlist *out_sg,
677 int *chunk, bool *zc)
679 struct tls_context *tls_ctx = tls_get_ctx(sk);
680 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
681 struct strp_msg *rxm = strp_msg(skb);
682 int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0;
683 struct aead_request *aead_req;
684 struct sk_buff *unused;
685 u8 *aad, *iv, *mem = NULL;
686 struct scatterlist *sgin = NULL;
687 struct scatterlist *sgout = NULL;
688 const int data_len = rxm->full_len - tls_ctx->rx.overhead_size;
690 if (*zc && (out_iov || out_sg)) {
692 n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1;
694 n_sgout = sg_nents(out_sg);
700 n_sgin = skb_cow_data(skb, 0, &unused);
704 /* Increment to accommodate AAD */
707 nsg = n_sgin + n_sgout;
709 aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv);
710 mem_size = aead_size + (nsg * sizeof(struct scatterlist));
711 mem_size = mem_size + TLS_AAD_SPACE_SIZE;
712 mem_size = mem_size + crypto_aead_ivsize(ctx->aead_recv);
714 /* Allocate a single block of memory which contains
715 * aead_req || sgin[] || sgout[] || aad || iv.
716 * This order achieves correct alignment for aead_req, sgin, sgout.
718 mem = kmalloc(mem_size, sk->sk_allocation);
722 /* Segment the allocated memory */
723 aead_req = (struct aead_request *)mem;
724 sgin = (struct scatterlist *)(mem + aead_size);
725 sgout = sgin + n_sgin;
726 aad = (u8 *)(sgout + n_sgout);
727 iv = aad + TLS_AAD_SPACE_SIZE;
730 err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE,
731 iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
732 tls_ctx->rx.iv_size);
737 memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
740 tls_make_aad(aad, rxm->full_len - tls_ctx->rx.overhead_size,
741 tls_ctx->rx.rec_seq, tls_ctx->rx.rec_seq_size,
745 sg_init_table(sgin, n_sgin);
746 sg_set_buf(&sgin[0], aad, TLS_AAD_SPACE_SIZE);
747 err = skb_to_sgvec(skb, &sgin[1],
748 rxm->offset + tls_ctx->rx.prepend_size,
749 rxm->full_len - tls_ctx->rx.prepend_size);
757 sg_init_table(sgout, n_sgout);
758 sg_set_buf(&sgout[0], aad, TLS_AAD_SPACE_SIZE);
761 err = zerocopy_from_iter(sk, out_iov, data_len, &pages,
763 (n_sgout - 1), false);
765 goto fallback_to_reg_recv;
767 memcpy(sgout, out_sg, n_sgout * sizeof(*sgout));
769 goto fallback_to_reg_recv;
772 fallback_to_reg_recv:
779 /* Prepare and submit AEAD request */
780 err = tls_do_decryption(sk, sgin, sgout, iv, data_len, aead_req);
782 /* Release the pages in case iov was mapped to pages */
783 for (; pages > 0; pages--)
784 put_page(sg_page(&sgout[pages]));
790 static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb,
791 struct iov_iter *dest, int *chunk, bool *zc)
793 struct tls_context *tls_ctx = tls_get_ctx(sk);
794 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
795 struct strp_msg *rxm = strp_msg(skb);
798 #ifdef CONFIG_TLS_DEVICE
799 err = tls_device_decrypted(sk, skb);
803 if (!ctx->decrypted) {
804 err = decrypt_internal(sk, skb, dest, NULL, chunk, zc);
811 rxm->offset += tls_ctx->rx.prepend_size;
812 rxm->full_len -= tls_ctx->rx.overhead_size;
813 tls_advance_record_sn(sk, &tls_ctx->rx);
814 ctx->decrypted = true;
815 ctx->saved_data_ready(sk);
820 int decrypt_skb(struct sock *sk, struct sk_buff *skb,
821 struct scatterlist *sgout)
826 return decrypt_internal(sk, skb, NULL, sgout, &chunk, &zc);
829 static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb,
832 struct tls_context *tls_ctx = tls_get_ctx(sk);
833 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
834 struct strp_msg *rxm = strp_msg(skb);
836 if (len < rxm->full_len) {
838 rxm->full_len -= len;
843 /* Finished with message */
844 ctx->recv_pkt = NULL;
846 __strp_unpause(&ctx->strp);
851 int tls_sw_recvmsg(struct sock *sk,
858 struct tls_context *tls_ctx = tls_get_ctx(sk);
859 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
860 unsigned char control;
861 struct strp_msg *rxm;
867 bool is_kvec = msg->msg_iter.type & ITER_KVEC;
871 if (unlikely(flags & MSG_ERRQUEUE))
872 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR);
876 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
877 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
882 skb = tls_wait_data(sk, flags, timeo, &err);
890 cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
891 sizeof(ctx->control), &ctx->control);
893 control = ctx->control;
894 if (ctx->control != TLS_RECORD_TYPE_DATA) {
895 if (cerr || msg->msg_flags & MSG_CTRUNC) {
900 } else if (control != ctx->control) {
904 if (!ctx->decrypted) {
905 int to_copy = rxm->full_len - tls_ctx->rx.overhead_size;
907 if (!is_kvec && to_copy <= len &&
908 likely(!(flags & MSG_PEEK)))
911 err = decrypt_skb_update(sk, skb, &msg->msg_iter,
914 tls_err_abort(sk, EBADMSG);
917 ctx->decrypted = true;
921 chunk = min_t(unsigned int, rxm->full_len, len);
922 err = skb_copy_datagram_msg(skb, rxm->offset, msg,
930 if (likely(!(flags & MSG_PEEK))) {
931 u8 control = ctx->control;
933 if (tls_sw_advance_skb(sk, skb, chunk)) {
934 /* Return full control message to
935 * userspace before trying to parse
936 * another message type
938 msg->msg_flags |= MSG_EOR;
939 if (control != TLS_RECORD_TYPE_DATA)
943 /* MSG_PEEK right now cannot look beyond current skb
944 * from strparser, meaning we cannot advance skb here
945 * and thus unpause strparser since we'd loose original
951 /* If we have a new message from strparser, continue now. */
952 if (copied >= target && !ctx->recv_pkt)
958 return copied ? : err;
961 ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
962 struct pipe_inode_info *pipe,
963 size_t len, unsigned int flags)
965 struct tls_context *tls_ctx = tls_get_ctx(sock->sk);
966 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
967 struct strp_msg *rxm = NULL;
968 struct sock *sk = sock->sk;
978 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
980 skb = tls_wait_data(sk, flags, timeo, &err);
982 goto splice_read_end;
984 /* splice does not support reading control messages */
985 if (ctx->control != TLS_RECORD_TYPE_DATA) {
987 goto splice_read_end;
990 if (!ctx->decrypted) {
991 err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc);
994 tls_err_abort(sk, EBADMSG);
995 goto splice_read_end;
997 ctx->decrypted = true;
1001 chunk = min_t(unsigned int, rxm->full_len, len);
1002 copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags);
1004 goto splice_read_end;
1006 if (likely(!(flags & MSG_PEEK)))
1007 tls_sw_advance_skb(sk, skb, copied);
1011 return copied ? : err;
1014 unsigned int tls_sw_poll(struct file *file, struct socket *sock,
1015 struct poll_table_struct *wait)
1018 struct sock *sk = sock->sk;
1019 struct tls_context *tls_ctx = tls_get_ctx(sk);
1020 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1022 /* Grab POLLOUT and POLLHUP from the underlying socket */
1023 ret = ctx->sk_poll(file, sock, wait);
1025 /* Clear POLLIN bits, and set based on recv_pkt */
1026 ret &= ~(POLLIN | POLLRDNORM);
1028 ret |= POLLIN | POLLRDNORM;
1033 static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
1035 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
1036 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1037 char header[TLS_HEADER_SIZE + MAX_IV_SIZE];
1038 struct strp_msg *rxm = strp_msg(skb);
1039 size_t cipher_overhead;
1040 size_t data_len = 0;
1043 /* Verify that we have a full TLS header, or wait for more data */
1044 if (rxm->offset + tls_ctx->rx.prepend_size > skb->len)
1047 /* Sanity-check size of on-stack buffer. */
1048 if (WARN_ON(tls_ctx->rx.prepend_size > sizeof(header))) {
1053 /* Linearize header to local buffer */
1054 ret = skb_copy_bits(skb, rxm->offset, header, tls_ctx->rx.prepend_size);
1059 ctx->control = header[0];
1061 data_len = ((header[4] & 0xFF) | (header[3] << 8));
1063 cipher_overhead = tls_ctx->rx.tag_size + tls_ctx->rx.iv_size;
1065 if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead) {
1069 if (data_len < cipher_overhead) {
1074 if (header[1] != TLS_VERSION_MINOR(tls_ctx->crypto_recv.info.version) ||
1075 header[2] != TLS_VERSION_MAJOR(tls_ctx->crypto_recv.info.version)) {
1080 #ifdef CONFIG_TLS_DEVICE
1081 handle_device_resync(strp->sk, TCP_SKB_CB(skb)->seq + rxm->offset,
1082 *(u64*)tls_ctx->rx.rec_seq);
1084 return data_len + TLS_HEADER_SIZE;
1087 tls_err_abort(strp->sk, ret);
1092 static void tls_queue(struct strparser *strp, struct sk_buff *skb)
1094 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
1095 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1097 ctx->decrypted = false;
1099 ctx->recv_pkt = skb;
1102 ctx->saved_data_ready(strp->sk);
1105 static void tls_data_ready(struct sock *sk)
1107 struct tls_context *tls_ctx = tls_get_ctx(sk);
1108 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1110 strp_data_ready(&ctx->strp);
1113 void tls_sw_free_resources_tx(struct sock *sk)
1115 struct tls_context *tls_ctx = tls_get_ctx(sk);
1116 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
1118 crypto_free_aead(ctx->aead_send);
1119 tls_free_both_sg(sk);
1124 void tls_sw_release_resources_rx(struct sock *sk)
1126 struct tls_context *tls_ctx = tls_get_ctx(sk);
1127 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1129 kfree(tls_ctx->rx.rec_seq);
1130 kfree(tls_ctx->rx.iv);
1132 if (ctx->aead_recv) {
1133 kfree_skb(ctx->recv_pkt);
1134 ctx->recv_pkt = NULL;
1135 crypto_free_aead(ctx->aead_recv);
1136 strp_stop(&ctx->strp);
1137 write_lock_bh(&sk->sk_callback_lock);
1138 sk->sk_data_ready = ctx->saved_data_ready;
1139 write_unlock_bh(&sk->sk_callback_lock);
1141 strp_done(&ctx->strp);
1146 void tls_sw_free_resources_rx(struct sock *sk)
1148 struct tls_context *tls_ctx = tls_get_ctx(sk);
1149 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1151 tls_sw_release_resources_rx(sk);
1156 int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
1158 struct tls_crypto_info *crypto_info;
1159 struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
1160 struct tls_sw_context_tx *sw_ctx_tx = NULL;
1161 struct tls_sw_context_rx *sw_ctx_rx = NULL;
1162 struct cipher_context *cctx;
1163 struct crypto_aead **aead;
1164 struct strp_callbacks cb;
1165 u16 nonce_size, tag_size, iv_size, rec_seq_size;
1175 if (!ctx->priv_ctx_tx) {
1176 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL);
1181 ctx->priv_ctx_tx = sw_ctx_tx;
1184 (struct tls_sw_context_tx *)ctx->priv_ctx_tx;
1187 if (!ctx->priv_ctx_rx) {
1188 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL);
1193 ctx->priv_ctx_rx = sw_ctx_rx;
1196 (struct tls_sw_context_rx *)ctx->priv_ctx_rx;
1201 crypto_init_wait(&sw_ctx_tx->async_wait);
1202 crypto_info = &ctx->crypto_send.info;
1204 aead = &sw_ctx_tx->aead_send;
1206 crypto_init_wait(&sw_ctx_rx->async_wait);
1207 crypto_info = &ctx->crypto_recv.info;
1209 aead = &sw_ctx_rx->aead_recv;
1212 switch (crypto_info->cipher_type) {
1213 case TLS_CIPHER_AES_GCM_128: {
1214 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1215 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
1216 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1217 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
1218 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
1220 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
1222 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
1230 /* Sanity-check the IV size for stack allocations. */
1231 if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE) {
1236 cctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
1237 cctx->tag_size = tag_size;
1238 cctx->overhead_size = cctx->prepend_size + cctx->tag_size;
1239 cctx->iv_size = iv_size;
1240 cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
1246 memcpy(cctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
1247 memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
1248 cctx->rec_seq_size = rec_seq_size;
1249 cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL);
1250 if (!cctx->rec_seq) {
1256 sg_init_table(sw_ctx_tx->sg_encrypted_data,
1257 ARRAY_SIZE(sw_ctx_tx->sg_encrypted_data));
1258 sg_init_table(sw_ctx_tx->sg_plaintext_data,
1259 ARRAY_SIZE(sw_ctx_tx->sg_plaintext_data));
1261 sg_init_table(sw_ctx_tx->sg_aead_in, 2);
1262 sg_set_buf(&sw_ctx_tx->sg_aead_in[0], sw_ctx_tx->aad_space,
1263 sizeof(sw_ctx_tx->aad_space));
1264 sg_unmark_end(&sw_ctx_tx->sg_aead_in[1]);
1265 sg_chain(sw_ctx_tx->sg_aead_in, 2,
1266 sw_ctx_tx->sg_plaintext_data);
1267 sg_init_table(sw_ctx_tx->sg_aead_out, 2);
1268 sg_set_buf(&sw_ctx_tx->sg_aead_out[0], sw_ctx_tx->aad_space,
1269 sizeof(sw_ctx_tx->aad_space));
1270 sg_unmark_end(&sw_ctx_tx->sg_aead_out[1]);
1271 sg_chain(sw_ctx_tx->sg_aead_out, 2,
1272 sw_ctx_tx->sg_encrypted_data);
1276 *aead = crypto_alloc_aead("gcm(aes)", 0, 0);
1277 if (IS_ERR(*aead)) {
1278 rc = PTR_ERR(*aead);
1284 ctx->push_pending_record = tls_sw_push_pending_record;
1286 rc = crypto_aead_setkey(*aead, gcm_128_info->key,
1287 TLS_CIPHER_AES_GCM_128_KEY_SIZE);
1291 rc = crypto_aead_setauthsize(*aead, cctx->tag_size);
1296 /* Set up strparser */
1297 memset(&cb, 0, sizeof(cb));
1298 cb.rcv_msg = tls_queue;
1299 cb.parse_msg = tls_read_size;
1301 strp_init(&sw_ctx_rx->strp, sk, &cb);
1303 write_lock_bh(&sk->sk_callback_lock);
1304 sw_ctx_rx->saved_data_ready = sk->sk_data_ready;
1305 sk->sk_data_ready = tls_data_ready;
1306 write_unlock_bh(&sk->sk_callback_lock);
1308 sw_ctx_rx->sk_poll = sk->sk_socket->ops->poll;
1310 strp_check_rcv(&sw_ctx_rx->strp);
1316 crypto_free_aead(*aead);
1319 kfree(cctx->rec_seq);
1320 cctx->rec_seq = NULL;
1326 kfree(ctx->priv_ctx_tx);
1327 ctx->priv_ctx_tx = NULL;
1329 kfree(ctx->priv_ctx_rx);
1330 ctx->priv_ctx_rx = NULL;