1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */
4 #include <linux/skbuff.h>
5 #include <linux/workqueue.h>
6 #include <net/strparser.h>
13 static struct workqueue_struct *tls_strp_wq;
15 static void tls_strp_abort_strp(struct tls_strparser *strp, int err)
22 /* Report an error on the lower socket */
23 WRITE_ONCE(strp->sk->sk_err, -err);
24 /* Paired with smp_rmb() in tcp_poll() */
26 sk_error_report(strp->sk);
29 static void tls_strp_anchor_free(struct tls_strparser *strp)
31 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
33 DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
35 shinfo->frag_list = NULL;
36 consume_skb(strp->anchor);
40 static struct sk_buff *
41 tls_strp_skb_copy(struct tls_strparser *strp, struct sk_buff *in_skb,
47 skb = alloc_skb_with_frags(0, len, TLS_PAGE_ORDER,
48 &err, strp->sk->sk_allocation);
52 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
53 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
55 WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
56 skb_frag_address(frag),
57 skb_frag_size(frag)));
58 offset += skb_frag_size(frag);
63 skb_copy_header(skb, in_skb);
67 /* Create a new skb with the contents of input copied to its page frags */
68 static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp)
73 skb = tls_strp_skb_copy(strp, strp->anchor, strp->stm.offset,
83 /* Steal the input skb, input msg is invalid after calling this function */
84 struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx)
86 struct tls_strparser *strp = &ctx->strp;
88 #ifdef CONFIG_TLS_DEVICE
89 DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted);
91 /* This function turns an input into an output,
92 * that can only happen if we have offload.
97 if (strp->copy_mode) {
100 /* Replace anchor with an empty skb, this is a little
101 * dangerous but __tls_cur_msg() warns on empty skbs
102 * so hopefully we'll catch abuses.
104 skb = alloc_skb(0, strp->sk->sk_allocation);
108 swap(strp->anchor, skb);
112 return tls_strp_msg_make_copy(strp);
115 /* Force the input skb to be in copy mode. The data ownership remains
116 * with the input skb itself (meaning unpause will wipe it) but it can
119 int tls_strp_msg_cow(struct tls_sw_context_rx *ctx)
121 struct tls_strparser *strp = &ctx->strp;
127 skb = tls_strp_msg_make_copy(strp);
131 tls_strp_anchor_free(strp);
134 tcp_read_done(strp->sk, strp->stm.full_len);
140 /* Make a clone (in the skb sense) of the input msg to keep a reference
141 * to the underlying data. The reference-holding skbs get placed on
144 int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst)
146 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
148 if (strp->copy_mode) {
151 WARN_ON_ONCE(!shinfo->nr_frags);
153 /* We can't skb_clone() the anchor, it gets wiped by unpause */
154 skb = alloc_skb(0, strp->sk->sk_allocation);
158 __skb_queue_tail(dst, strp->anchor);
161 struct sk_buff *iter, *clone;
162 int chunk, len, offset;
164 offset = strp->stm.offset;
165 len = strp->stm.full_len;
166 iter = shinfo->frag_list;
169 if (iter->len <= offset) {
174 chunk = iter->len - offset;
177 clone = skb_clone(iter, strp->sk->sk_allocation);
180 __skb_queue_tail(dst, clone);
191 static void tls_strp_flush_anchor_copy(struct tls_strparser *strp)
193 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
196 DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
198 for (i = 0; i < shinfo->nr_frags; i++)
199 __skb_frag_unref(&shinfo->frags[i], false);
200 shinfo->nr_frags = 0;
201 if (strp->copy_mode) {
202 kfree_skb_list(shinfo->frag_list);
203 shinfo->frag_list = NULL;
206 strp->mixed_decrypted = 0;
209 static int tls_strp_copyin_frag(struct tls_strparser *strp, struct sk_buff *skb,
210 struct sk_buff *in_skb, unsigned int offset,
217 frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE];
220 /* First make sure we got the header */
221 if (!strp->stm.full_len) {
222 /* Assume one page is more than enough for headers */
223 chunk = min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag));
224 WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
225 skb_frag_address(frag) +
230 skb->data_len += chunk;
231 skb_frag_size_add(frag, chunk);
233 sz = tls_rx_msg_size(strp, skb);
237 /* We may have over-read, sz == 0 is guaranteed under-read */
238 if (unlikely(sz && sz < skb->len)) {
239 int over = skb->len - sz;
241 WARN_ON_ONCE(over > chunk);
243 skb->data_len -= over;
244 skb_frag_size_add(frag, -over);
253 strp->stm.full_len = sz;
254 if (!strp->stm.full_len)
258 /* Load up more data */
259 while (len && strp->stm.full_len > skb->len) {
260 chunk = min_t(size_t, len, strp->stm.full_len - skb->len);
261 chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag));
262 WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
263 skb_frag_address(frag) +
268 skb->data_len += chunk;
269 skb_frag_size_add(frag, chunk);
279 static int tls_strp_copyin_skb(struct tls_strparser *strp, struct sk_buff *skb,
280 struct sk_buff *in_skb, unsigned int offset,
283 struct sk_buff *nskb, *first, *last;
284 struct skb_shared_info *shinfo;
288 if (strp->stm.full_len)
289 chunk = strp->stm.full_len - skb->len;
291 chunk = TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
292 chunk = min(chunk, in_len);
294 nskb = tls_strp_skb_copy(strp, in_skb, offset, chunk);
298 shinfo = skb_shinfo(skb);
299 if (!shinfo->frag_list) {
300 shinfo->frag_list = nskb;
303 first = shinfo->frag_list;
310 skb->data_len += chunk;
312 if (!strp->stm.full_len) {
313 sz = tls_rx_msg_size(strp, skb);
317 /* We may have over-read, sz == 0 is guaranteed under-read */
318 if (unlikely(sz && sz < skb->len)) {
319 int over = skb->len - sz;
321 WARN_ON_ONCE(over > chunk);
323 skb->data_len -= over;
324 __pskb_trim(nskb, nskb->len - over);
329 strp->stm.full_len = sz;
335 static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb,
336 unsigned int offset, size_t in_len)
338 struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data;
347 skb_copy_decrypted(skb, in_skb);
349 strp->mixed_decrypted |= !!skb_cmp_decrypted(skb, in_skb);
351 if (IS_ENABLED(CONFIG_TLS_DEVICE) && strp->mixed_decrypted)
352 ret = tls_strp_copyin_skb(strp, skb, in_skb, offset, in_len);
354 ret = tls_strp_copyin_frag(strp, skb, in_skb, offset, in_len);
360 if (strp->stm.full_len && strp->stm.full_len == skb->len) {
363 WRITE_ONCE(strp->msg_ready, 1);
364 tls_rx_msg_ready(strp);
370 static int tls_strp_read_copyin(struct tls_strparser *strp)
372 read_descriptor_t desc;
374 desc.arg.data = strp;
376 desc.count = 1; /* give more than one skb per call */
378 /* sk should be locked here, so okay to do read_sock */
379 tcp_read_sock(strp->sk, &desc, tls_strp_copyin);
384 static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort)
386 struct skb_shared_info *shinfo;
390 /* If the rbuf is small or rcv window has collapsed to 0 we need
391 * to read the data out. Otherwise the connection will stall.
392 * Without pressure threshold of INT_MAX will never be ready.
394 if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX)))
397 shinfo = skb_shinfo(strp->anchor);
398 shinfo->frag_list = NULL;
400 /* If we don't know the length go max plus page for cipher overhead */
401 need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
403 for (len = need_spc; len > 0; len -= PAGE_SIZE) {
404 page = alloc_page(strp->sk->sk_allocation);
406 tls_strp_flush_anchor_copy(strp);
410 skb_fill_page_desc(strp->anchor, shinfo->nr_frags++,
415 strp->stm.offset = 0;
417 strp->anchor->len = 0;
418 strp->anchor->data_len = 0;
419 strp->anchor->truesize = round_up(need_spc, PAGE_SIZE);
421 tls_strp_read_copyin(strp);
426 static bool tls_strp_check_queue_ok(struct tls_strparser *strp)
428 unsigned int len = strp->stm.offset + strp->stm.full_len;
429 struct sk_buff *first, *skb;
432 first = skb_shinfo(strp->anchor)->frag_list;
434 seq = TCP_SKB_CB(first)->seq;
436 /* Make sure there's no duplicate data in the queue,
437 * and the decrypted status matches.
439 while (skb->len < len) {
444 if (TCP_SKB_CB(skb)->seq != seq)
446 if (skb_cmp_decrypted(first, skb))
453 static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len)
455 struct tcp_sock *tp = tcp_sk(strp->sk);
456 struct sk_buff *first;
459 first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset);
460 if (WARN_ON_ONCE(!first))
463 /* Bestow the state onto the anchor */
464 strp->anchor->len = offset + len;
465 strp->anchor->data_len = offset + len;
466 strp->anchor->truesize = offset + len;
468 skb_shinfo(strp->anchor)->frag_list = first;
470 skb_copy_header(strp->anchor, first);
471 strp->anchor->destructor = NULL;
473 strp->stm.offset = offset;
476 void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh)
478 struct strp_msg *rxm;
481 DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready);
482 DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len);
484 if (!strp->copy_mode && force_refresh) {
485 if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len))
488 tls_strp_load_anchor_with_queue(strp, strp->stm.full_len);
491 rxm = strp_msg(strp->anchor);
492 rxm->full_len = strp->stm.full_len;
493 rxm->offset = strp->stm.offset;
494 tlm = tls_msg(strp->anchor);
495 tlm->control = strp->mark;
498 /* Called with lock held on lower socket */
499 static int tls_strp_read_sock(struct tls_strparser *strp)
503 inq = tcp_inq(strp->sk);
507 if (unlikely(strp->copy_mode))
508 return tls_strp_read_copyin(strp);
510 if (inq < strp->stm.full_len)
511 return tls_strp_read_copy(strp, true);
513 if (!strp->stm.full_len) {
514 tls_strp_load_anchor_with_queue(strp, inq);
516 sz = tls_rx_msg_size(strp, strp->anchor);
518 tls_strp_abort_strp(strp, sz);
522 strp->stm.full_len = sz;
524 if (!strp->stm.full_len || inq < strp->stm.full_len)
525 return tls_strp_read_copy(strp, true);
528 if (!tls_strp_check_queue_ok(strp))
529 return tls_strp_read_copy(strp, false);
531 WRITE_ONCE(strp->msg_ready, 1);
532 tls_rx_msg_ready(strp);
537 void tls_strp_check_rcv(struct tls_strparser *strp)
539 if (unlikely(strp->stopped) || strp->msg_ready)
542 if (tls_strp_read_sock(strp) == -ENOMEM)
543 queue_work(tls_strp_wq, &strp->work);
546 /* Lower sock lock held */
547 void tls_strp_data_ready(struct tls_strparser *strp)
549 /* This check is needed to synchronize with do_tls_strp_work.
550 * do_tls_strp_work acquires a process lock (lock_sock) whereas
551 * the lock held here is bh_lock_sock. The two locks can be
552 * held by different threads at the same time, but bh_lock_sock
553 * allows a thread in BH context to safely check if the process
554 * lock is held. In this case, if the lock is held, queue work.
556 if (sock_owned_by_user_nocheck(strp->sk)) {
557 queue_work(tls_strp_wq, &strp->work);
561 tls_strp_check_rcv(strp);
564 static void tls_strp_work(struct work_struct *w)
566 struct tls_strparser *strp =
567 container_of(w, struct tls_strparser, work);
570 tls_strp_check_rcv(strp);
571 release_sock(strp->sk);
574 void tls_strp_msg_done(struct tls_strparser *strp)
576 WARN_ON(!strp->stm.full_len);
578 if (likely(!strp->copy_mode))
579 tcp_read_done(strp->sk, strp->stm.full_len);
581 tls_strp_flush_anchor_copy(strp);
583 WRITE_ONCE(strp->msg_ready, 0);
584 memset(&strp->stm, 0, sizeof(strp->stm));
586 tls_strp_check_rcv(strp);
589 void tls_strp_stop(struct tls_strparser *strp)
594 int tls_strp_init(struct tls_strparser *strp, struct sock *sk)
596 memset(strp, 0, sizeof(*strp));
600 strp->anchor = alloc_skb(0, GFP_KERNEL);
604 INIT_WORK(&strp->work, tls_strp_work);
609 /* strp must already be stopped so that tls_strp_recv will no longer be called.
610 * Note that tls_strp_done is not called with the lower socket held.
612 void tls_strp_done(struct tls_strparser *strp)
614 WARN_ON(!strp->stopped);
616 cancel_work_sync(&strp->work);
617 tls_strp_anchor_free(strp);
620 int __init tls_strp_dev_init(void)
622 tls_strp_wq = create_workqueue("tls-strp");
623 if (unlikely(!tls_strp_wq))
629 void tls_strp_dev_exit(void)
631 destroy_workqueue(tls_strp_wq);