1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Routines having to do with the 'struct sk_buff' memory handlers.
5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
6 * Florian La Roche <rzsfl@rz.uni-sb.de>
9 * Alan Cox : Fixed the worst of the load
11 * Dave Platt : Interrupt stacking fix.
12 * Richard Kooijman : Timestamp fixes.
13 * Alan Cox : Changed buffer format.
14 * Alan Cox : destructor hook for AF_UNIX etc.
15 * Linus Torvalds : Better skb_clone.
16 * Alan Cox : Added skb_copy.
17 * Alan Cox : Added all the changed routines Linus
18 * only put in the headers
19 * Ray VanTassle : Fixed --skb->lock in free
20 * Alan Cox : skb_copy copy arp field
21 * Andi Kleen : slabified it.
22 * Robert Olsson : Removed skb_head_pool
25 * The __skb_ routines should be called with interrupts
26 * disabled, or you better be *real* sure that the operation is atomic
27 * with respect to whatever list is being frobbed (e.g. via lock_sock()
28 * or via disabling bottom half handlers, etc).
32 * The functions in this file will not compile correctly with gcc 2.4.x
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
37 #include <linux/module.h>
38 #include <linux/types.h>
39 #include <linux/kernel.h>
41 #include <linux/interrupt.h>
43 #include <linux/inet.h>
44 #include <linux/slab.h>
45 #include <linux/tcp.h>
46 #include <linux/udp.h>
47 #include <linux/sctp.h>
48 #include <linux/netdevice.h>
49 #ifdef CONFIG_NET_CLS_ACT
50 #include <net/pkt_sched.h>
52 #include <linux/string.h>
53 #include <linux/skbuff.h>
54 #include <linux/splice.h>
55 #include <linux/cache.h>
56 #include <linux/rtnetlink.h>
57 #include <linux/init.h>
58 #include <linux/scatterlist.h>
59 #include <linux/errqueue.h>
60 #include <linux/prefetch.h>
61 #include <linux/bitfield.h>
62 #include <linux/if_vlan.h>
63 #include <linux/mpls.h>
64 #include <linux/kcov.h>
65 #include <linux/iov_iter.h>
67 #include <net/protocol.h>
70 #include <net/checksum.h>
72 #include <net/ip6_checksum.h>
75 #include <net/mptcp.h>
77 #include <net/page_pool/helpers.h>
78 #include <net/dropreason.h>
80 #include <linux/uaccess.h>
81 #include <trace/events/skb.h>
82 #include <linux/highmem.h>
83 #include <linux/capability.h>
84 #include <linux/user_namespace.h>
85 #include <linux/indirect_call_wrapper.h>
86 #include <linux/textsearch.h>
89 #include "sock_destructor.h"
91 struct kmem_cache *skbuff_cache __ro_after_init;
92 static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
93 #ifdef CONFIG_SKB_EXTENSIONS
94 static struct kmem_cache *skbuff_ext_cache __ro_after_init;
98 static struct kmem_cache *skb_small_head_cache __ro_after_init;
100 #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER)
102 /* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two.
103 * This should ensure that SKB_SMALL_HEAD_HEADROOM is a unique
104 * size, and we can differentiate heads from skb_small_head_cache
105 * vs system slabs by looking at their size (skb_end_offset()).
107 #define SKB_SMALL_HEAD_CACHE_SIZE \
108 (is_power_of_2(SKB_SMALL_HEAD_SIZE) ? \
109 (SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) : \
112 #define SKB_SMALL_HEAD_HEADROOM \
113 SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE)
115 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
116 EXPORT_SYMBOL(sysctl_max_skb_frags);
119 #define FN(reason) [SKB_DROP_REASON_##reason] = #reason,
120 static const char * const drop_reasons[] = {
121 [SKB_CONSUMED] = "CONSUMED",
122 DEFINE_DROP_REASON(FN, FN)
125 static const struct drop_reason_list drop_reasons_core = {
126 .reasons = drop_reasons,
127 .n_reasons = ARRAY_SIZE(drop_reasons),
130 const struct drop_reason_list __rcu *
131 drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = {
132 [SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core),
134 EXPORT_SYMBOL(drop_reasons_by_subsys);
137 * drop_reasons_register_subsys - register another drop reason subsystem
138 * @subsys: the subsystem to register, must not be the core
139 * @list: the list of drop reasons within the subsystem, must point to
140 * a statically initialized list
142 void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys,
143 const struct drop_reason_list *list)
145 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
146 subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
147 "invalid subsystem %d\n", subsys))
150 /* must point to statically allocated memory, so INIT is OK */
151 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list);
153 EXPORT_SYMBOL_GPL(drop_reasons_register_subsys);
156 * drop_reasons_unregister_subsys - unregister a drop reason subsystem
157 * @subsys: the subsystem to remove, must not be the core
159 * Note: This will synchronize_rcu() to ensure no users when it returns.
161 void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys)
163 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
164 subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
165 "invalid subsystem %d\n", subsys))
168 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL);
172 EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys);
175 * skb_panic - private function for out-of-line support
179 * @msg: skb_over_panic or skb_under_panic
181 * Out-of-line support for skb_put() and skb_push().
182 * Called via the wrapper skb_over_panic() or skb_under_panic().
183 * Keep out of line to prevent kernel bloat.
184 * __builtin_return_address is not used because it is not always reliable.
186 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
189 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
190 msg, addr, skb->len, sz, skb->head, skb->data,
191 (unsigned long)skb->tail, (unsigned long)skb->end,
192 skb->dev ? skb->dev->name : "<NULL>");
196 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
198 skb_panic(skb, sz, addr, __func__);
201 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
203 skb_panic(skb, sz, addr, __func__);
206 #define NAPI_SKB_CACHE_SIZE 64
207 #define NAPI_SKB_CACHE_BULK 16
208 #define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2)
210 #if PAGE_SIZE == SZ_4K
212 #define NAPI_HAS_SMALL_PAGE_FRAG 1
213 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) ((nc).pfmemalloc)
215 /* specialized page frag allocator using a single order 0 page
216 * and slicing it into 1K sized fragment. Constrained to systems
217 * with a very limited amount of 1K fragments fitting a single
218 * page - to avoid excessive truesize underestimation
221 struct page_frag_1k {
227 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp)
232 offset = nc->offset - SZ_1K;
233 if (likely(offset >= 0))
236 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
240 nc->va = page_address(page);
241 nc->pfmemalloc = page_is_pfmemalloc(page);
242 offset = PAGE_SIZE - SZ_1K;
243 page_ref_add(page, offset / SZ_1K);
247 return nc->va + offset;
251 /* the small page is actually unused in this build; add dummy helpers
252 * to please the compiler and avoid later preprocessor's conditionals
254 #define NAPI_HAS_SMALL_PAGE_FRAG 0
255 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) false
257 struct page_frag_1k {
260 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp_mask)
267 struct napi_alloc_cache {
268 struct page_frag_cache page;
269 struct page_frag_1k page_small;
270 unsigned int skb_count;
271 void *skb_cache[NAPI_SKB_CACHE_SIZE];
274 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
275 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
277 /* Double check that napi_get_frags() allocates skbs with
278 * skb->head being backed by slab, not a page fragment.
279 * This is to make sure bug fixed in 3226b158e67c
280 * ("net: avoid 32 x truesize under-estimation for tiny skbs")
281 * does not accidentally come back.
283 void napi_get_frags_check(struct napi_struct *napi)
288 skb = napi_get_frags(napi);
289 WARN_ON_ONCE(!NAPI_HAS_SMALL_PAGE_FRAG && skb && skb->head_frag);
290 napi_free_frags(napi);
294 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
296 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
298 fragsz = SKB_DATA_ALIGN(fragsz);
300 return page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask);
302 EXPORT_SYMBOL(__napi_alloc_frag_align);
304 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
308 fragsz = SKB_DATA_ALIGN(fragsz);
309 if (in_hardirq() || irqs_disabled()) {
310 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache);
312 data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask);
314 struct napi_alloc_cache *nc;
317 nc = this_cpu_ptr(&napi_alloc_cache);
318 data = page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask);
323 EXPORT_SYMBOL(__netdev_alloc_frag_align);
325 static struct sk_buff *napi_skb_cache_get(void)
327 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
330 if (unlikely(!nc->skb_count)) {
331 nc->skb_count = kmem_cache_alloc_bulk(skbuff_cache,
335 if (unlikely(!nc->skb_count))
339 skb = nc->skb_cache[--nc->skb_count];
340 kasan_mempool_unpoison_object(skb, kmem_cache_size(skbuff_cache));
345 static inline void __finalize_skb_around(struct sk_buff *skb, void *data,
348 struct skb_shared_info *shinfo;
350 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
352 /* Assumes caller memset cleared SKB */
353 skb->truesize = SKB_TRUESIZE(size);
354 refcount_set(&skb->users, 1);
357 skb_reset_tail_pointer(skb);
358 skb_set_end_offset(skb, size);
359 skb->mac_header = (typeof(skb->mac_header))~0U;
360 skb->transport_header = (typeof(skb->transport_header))~0U;
361 skb->alloc_cpu = raw_smp_processor_id();
362 /* make sure we initialize shinfo sequentially */
363 shinfo = skb_shinfo(skb);
364 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
365 atomic_set(&shinfo->dataref, 1);
367 skb_set_kcov_handle(skb, kcov_common_handle());
370 static inline void *__slab_build_skb(struct sk_buff *skb, void *data,
375 /* Must find the allocation size (and grow it to match). */
377 /* krealloc() will immediately return "data" when
378 * "ksize(data)" is requested: it is the existing upper
379 * bounds. As a result, GFP_ATOMIC will be ignored. Note
380 * that this "new" pointer needs to be passed back to the
381 * caller for use so the __alloc_size hinting will be
384 resized = krealloc(data, *size, GFP_ATOMIC);
385 WARN_ON_ONCE(resized != data);
389 /* build_skb() variant which can operate on slab buffers.
390 * Note that this should be used sparingly as slab buffers
391 * cannot be combined efficiently by GRO!
393 struct sk_buff *slab_build_skb(void *data)
398 skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);
402 memset(skb, 0, offsetof(struct sk_buff, tail));
403 data = __slab_build_skb(skb, data, &size);
404 __finalize_skb_around(skb, data, size);
408 EXPORT_SYMBOL(slab_build_skb);
410 /* Caller must provide SKB that is memset cleared */
411 static void __build_skb_around(struct sk_buff *skb, void *data,
412 unsigned int frag_size)
414 unsigned int size = frag_size;
416 /* frag_size == 0 is considered deprecated now. Callers
417 * using slab buffer should use slab_build_skb() instead.
419 if (WARN_ONCE(size == 0, "Use slab_build_skb() instead"))
420 data = __slab_build_skb(skb, data, &size);
422 __finalize_skb_around(skb, data, size);
426 * __build_skb - build a network buffer
427 * @data: data buffer provided by caller
428 * @frag_size: size of data (must not be 0)
430 * Allocate a new &sk_buff. Caller provides space holding head and
431 * skb_shared_info. @data must have been allocated from the page
432 * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc()
433 * allocation is deprecated, and callers should use slab_build_skb()
435 * The return is the new skb buffer.
436 * On a failure the return is %NULL, and @data is not freed.
438 * Before IO, driver allocates only data buffer where NIC put incoming frame
439 * Driver should add room at head (NET_SKB_PAD) and
440 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
441 * After IO, driver calls build_skb(), to allocate sk_buff and populate it
442 * before giving packet to stack.
443 * RX rings only contains data buffers, not full skbs.
445 struct sk_buff *__build_skb(void *data, unsigned int frag_size)
449 skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);
453 memset(skb, 0, offsetof(struct sk_buff, tail));
454 __build_skb_around(skb, data, frag_size);
459 /* build_skb() is wrapper over __build_skb(), that specifically
460 * takes care of skb->head and skb->pfmemalloc
462 struct sk_buff *build_skb(void *data, unsigned int frag_size)
464 struct sk_buff *skb = __build_skb(data, frag_size);
466 if (likely(skb && frag_size)) {
468 skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
472 EXPORT_SYMBOL(build_skb);
475 * build_skb_around - build a network buffer around provided skb
476 * @skb: sk_buff provide by caller, must be memset cleared
477 * @data: data buffer provided by caller
478 * @frag_size: size of data
480 struct sk_buff *build_skb_around(struct sk_buff *skb,
481 void *data, unsigned int frag_size)
486 __build_skb_around(skb, data, frag_size);
490 skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
494 EXPORT_SYMBOL(build_skb_around);
497 * __napi_build_skb - build a network buffer
498 * @data: data buffer provided by caller
499 * @frag_size: size of data
501 * Version of __build_skb() that uses NAPI percpu caches to obtain
502 * skbuff_head instead of inplace allocation.
504 * Returns a new &sk_buff on success, %NULL on allocation failure.
506 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size)
510 skb = napi_skb_cache_get();
514 memset(skb, 0, offsetof(struct sk_buff, tail));
515 __build_skb_around(skb, data, frag_size);
521 * napi_build_skb - build a network buffer
522 * @data: data buffer provided by caller
523 * @frag_size: size of data
525 * Version of __napi_build_skb() that takes care of skb->head_frag
526 * and skb->pfmemalloc when the data is a page or page fragment.
528 * Returns a new &sk_buff on success, %NULL on allocation failure.
530 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size)
532 struct sk_buff *skb = __napi_build_skb(data, frag_size);
534 if (likely(skb) && frag_size) {
536 skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
541 EXPORT_SYMBOL(napi_build_skb);
544 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
545 * the caller if emergency pfmemalloc reserves are being used. If it is and
546 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
547 * may be used. Otherwise, the packet data may be discarded until enough
550 static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node,
553 bool ret_pfmemalloc = false;
557 obj_size = SKB_HEAD_ALIGN(*size);
558 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&
559 !(flags & KMALLOC_NOT_NORMAL_BITS)) {
560 obj = kmem_cache_alloc_node(skb_small_head_cache,
561 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
563 *size = SKB_SMALL_HEAD_CACHE_SIZE;
564 if (obj || !(gfp_pfmemalloc_allowed(flags)))
566 /* Try again but now we are using pfmemalloc reserves */
567 ret_pfmemalloc = true;
568 obj = kmem_cache_alloc_node(skb_small_head_cache, flags, node);
572 obj_size = kmalloc_size_roundup(obj_size);
573 /* The following cast might truncate high-order bits of obj_size, this
574 * is harmless because kmalloc(obj_size >= 2^32) will fail anyway.
576 *size = (unsigned int)obj_size;
579 * Try a regular allocation, when that fails and we're not entitled
580 * to the reserves, fail.
582 obj = kmalloc_node_track_caller(obj_size,
583 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
585 if (obj || !(gfp_pfmemalloc_allowed(flags)))
588 /* Try again but now we are using pfmemalloc reserves */
589 ret_pfmemalloc = true;
590 obj = kmalloc_node_track_caller(obj_size, flags, node);
594 *pfmemalloc = ret_pfmemalloc;
599 /* Allocate a new skbuff. We do this ourselves so we can fill in a few
600 * 'private' fields and also do memory statistics to find all the
606 * __alloc_skb - allocate a network buffer
607 * @size: size to allocate
608 * @gfp_mask: allocation mask
609 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
610 * instead of head cache and allocate a cloned (child) skb.
611 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
612 * allocations in case the data is required for writeback
613 * @node: numa node to allocate memory on
615 * Allocate a new &sk_buff. The returned buffer has no headroom and a
616 * tail room of at least size bytes. The object has a reference count
617 * of one. The return is the buffer. On a failure the return is %NULL.
619 * Buffers may only be allocated from interrupts using a @gfp_mask of
622 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
625 struct kmem_cache *cache;
630 cache = (flags & SKB_ALLOC_FCLONE)
631 ? skbuff_fclone_cache : skbuff_cache;
633 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
634 gfp_mask |= __GFP_MEMALLOC;
637 if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI &&
638 likely(node == NUMA_NO_NODE || node == numa_mem_id()))
639 skb = napi_skb_cache_get();
641 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node);
646 /* We do our best to align skb_shared_info on a separate cache
647 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
648 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
649 * Both skb->head and skb_shared_info are cache line aligned.
651 data = kmalloc_reserve(&size, gfp_mask, node, &pfmemalloc);
654 /* kmalloc_size_roundup() might give us more room than requested.
655 * Put skb_shared_info exactly at the end of allocated zone,
656 * to allow max possible filling before reallocation.
658 prefetchw(data + SKB_WITH_OVERHEAD(size));
661 * Only clear those fields we need to clear, not those that we will
662 * actually initialise below. Hence, don't put any more fields after
663 * the tail pointer in struct sk_buff!
665 memset(skb, 0, offsetof(struct sk_buff, tail));
666 __build_skb_around(skb, data, size);
667 skb->pfmemalloc = pfmemalloc;
669 if (flags & SKB_ALLOC_FCLONE) {
670 struct sk_buff_fclones *fclones;
672 fclones = container_of(skb, struct sk_buff_fclones, skb1);
674 skb->fclone = SKB_FCLONE_ORIG;
675 refcount_set(&fclones->fclone_ref, 1);
681 kmem_cache_free(cache, skb);
684 EXPORT_SYMBOL(__alloc_skb);
687 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
688 * @dev: network device to receive on
689 * @len: length to allocate
690 * @gfp_mask: get_free_pages mask, passed to alloc_skb
692 * Allocate a new &sk_buff and assign it a usage count of one. The
693 * buffer has NET_SKB_PAD headroom built in. Users should allocate
694 * the headroom they think they need without accounting for the
695 * built in space. The built in space is used for optimisations.
697 * %NULL is returned if there is no free memory.
699 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
702 struct page_frag_cache *nc;
709 /* If requested length is either too small or too big,
710 * we use kmalloc() for skb->head allocation.
712 if (len <= SKB_WITH_OVERHEAD(1024) ||
713 len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
714 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
715 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
721 len = SKB_HEAD_ALIGN(len);
723 if (sk_memalloc_socks())
724 gfp_mask |= __GFP_MEMALLOC;
726 if (in_hardirq() || irqs_disabled()) {
727 nc = this_cpu_ptr(&netdev_alloc_cache);
728 data = page_frag_alloc(nc, len, gfp_mask);
729 pfmemalloc = nc->pfmemalloc;
732 nc = this_cpu_ptr(&napi_alloc_cache.page);
733 data = page_frag_alloc(nc, len, gfp_mask);
734 pfmemalloc = nc->pfmemalloc;
741 skb = __build_skb(data, len);
742 if (unlikely(!skb)) {
752 skb_reserve(skb, NET_SKB_PAD);
758 EXPORT_SYMBOL(__netdev_alloc_skb);
761 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
762 * @napi: napi instance this buffer was allocated for
763 * @len: length to allocate
764 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
766 * Allocate a new sk_buff for use in NAPI receive. This buffer will
767 * attempt to allocate the head from a special reserved region used
768 * only for NAPI Rx allocation. By doing this we can save several
769 * CPU cycles by avoiding having to disable and re-enable IRQs.
771 * %NULL is returned if there is no free memory.
773 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
776 struct napi_alloc_cache *nc;
781 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
782 len += NET_SKB_PAD + NET_IP_ALIGN;
784 /* If requested length is either too small or too big,
785 * we use kmalloc() for skb->head allocation.
786 * When the small frag allocator is available, prefer it over kmalloc
787 * for small fragments
789 if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) ||
790 len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
791 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
792 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI,
799 nc = this_cpu_ptr(&napi_alloc_cache);
801 if (sk_memalloc_socks())
802 gfp_mask |= __GFP_MEMALLOC;
804 if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) {
805 /* we are artificially inflating the allocation size, but
806 * that is not as bad as it may look like, as:
807 * - 'len' less than GRO_MAX_HEAD makes little sense
808 * - On most systems, larger 'len' values lead to fragment
809 * size above 512 bytes
810 * - kmalloc would use the kmalloc-1k slab for such values
811 * - Builds with smaller GRO_MAX_HEAD will very likely do
812 * little networking, as that implies no WiFi and no
813 * tunnels support, and 32 bits arches.
817 data = page_frag_alloc_1k(&nc->page_small, gfp_mask);
818 pfmemalloc = NAPI_SMALL_PAGE_PFMEMALLOC(nc->page_small);
820 len = SKB_HEAD_ALIGN(len);
822 data = page_frag_alloc(&nc->page, len, gfp_mask);
823 pfmemalloc = nc->page.pfmemalloc;
829 skb = __napi_build_skb(data, len);
830 if (unlikely(!skb)) {
840 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
841 skb->dev = napi->dev;
846 EXPORT_SYMBOL(__napi_alloc_skb);
848 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
849 int size, unsigned int truesize)
851 DEBUG_NET_WARN_ON_ONCE(size > truesize);
853 skb_fill_page_desc(skb, i, page, off, size);
855 skb->data_len += size;
856 skb->truesize += truesize;
858 EXPORT_SYMBOL(skb_add_rx_frag);
860 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
861 unsigned int truesize)
863 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
865 DEBUG_NET_WARN_ON_ONCE(size > truesize);
867 skb_frag_size_add(frag, size);
869 skb->data_len += size;
870 skb->truesize += truesize;
872 EXPORT_SYMBOL(skb_coalesce_rx_frag);
874 static void skb_drop_list(struct sk_buff **listp)
876 kfree_skb_list(*listp);
880 static inline void skb_drop_fraglist(struct sk_buff *skb)
882 skb_drop_list(&skb_shinfo(skb)->frag_list);
885 static void skb_clone_fraglist(struct sk_buff *skb)
887 struct sk_buff *list;
889 skb_walk_frags(skb, list)
893 static bool is_pp_page(struct page *page)
895 return (page->pp_magic & ~0x3UL) == PP_SIGNATURE;
898 #if IS_ENABLED(CONFIG_PAGE_POOL)
899 bool napi_pp_put_page(struct page *page, bool napi_safe)
901 bool allow_direct = false;
902 struct page_pool *pp;
904 page = compound_head(page);
906 /* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation
907 * in order to preserve any existing bits, such as bit 0 for the
908 * head page of compound page and bit 1 for pfmemalloc page, so
909 * mask those bits for freeing side when doing below checking,
910 * and page_is_pfmemalloc() is checked in __page_pool_put_page()
911 * to avoid recycling the pfmemalloc page.
913 if (unlikely(!is_pp_page(page)))
918 /* Allow direct recycle if we have reasons to believe that we are
919 * in the same context as the consumer would run, so there's
921 * __page_pool_put_page() makes sure we're not in hardirq context
922 * and interrupts are enabled prior to accessing the cache.
924 if (napi_safe || in_softirq()) {
925 const struct napi_struct *napi = READ_ONCE(pp->p.napi);
927 allow_direct = napi &&
928 READ_ONCE(napi->list_owner) == smp_processor_id();
931 /* Driver set this to memory recycling info. Reset it on recycle.
932 * This will *not* work for NIC using a split-page memory model.
933 * The page will be returned to the pool here regardless of the
934 * 'flipped' fragment being in use or not.
936 page_pool_put_full_page(pp, page, allow_direct);
940 EXPORT_SYMBOL(napi_pp_put_page);
943 static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe)
945 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
947 return napi_pp_put_page(virt_to_page(data), napi_safe);
951 * skb_pp_frag_ref() - Increase fragment references of a page pool aware skb
952 * @skb: page pool aware skb
954 * Increase the fragment reference count (pp_ref_count) of a skb. This is
955 * intended to gain fragment references only for page pool aware skbs,
956 * i.e. when skb->pp_recycle is true, and not for fragments in a
957 * non-pp-recycling skb. It has a fallback to increase references on normal
958 * pages, as page pool aware skbs may also have normal page fragments.
960 static int skb_pp_frag_ref(struct sk_buff *skb)
962 struct skb_shared_info *shinfo;
963 struct page *head_page;
966 if (!skb->pp_recycle)
969 shinfo = skb_shinfo(skb);
971 for (i = 0; i < shinfo->nr_frags; i++) {
972 head_page = compound_head(skb_frag_page(&shinfo->frags[i]));
973 if (likely(is_pp_page(head_page)))
974 page_pool_ref_page(head_page);
976 page_ref_inc(head_page);
981 static void skb_kfree_head(void *head, unsigned int end_offset)
983 if (end_offset == SKB_SMALL_HEAD_HEADROOM)
984 kmem_cache_free(skb_small_head_cache, head);
989 static void skb_free_head(struct sk_buff *skb, bool napi_safe)
991 unsigned char *head = skb->head;
993 if (skb->head_frag) {
994 if (skb_pp_recycle(skb, head, napi_safe))
998 skb_kfree_head(head, skb_end_offset(skb));
1002 static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason,
1005 struct skb_shared_info *shinfo = skb_shinfo(skb);
1009 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
1013 if (skb_zcopy(skb)) {
1014 bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS;
1016 skb_zcopy_clear(skb, true);
1021 for (i = 0; i < shinfo->nr_frags; i++)
1022 napi_frag_unref(&shinfo->frags[i], skb->pp_recycle, napi_safe);
1025 if (shinfo->frag_list)
1026 kfree_skb_list_reason(shinfo->frag_list, reason);
1028 skb_free_head(skb, napi_safe);
1030 /* When we clone an SKB we copy the reycling bit. The pp_recycle
1031 * bit is only set on the head though, so in order to avoid races
1032 * while trying to recycle fragments on __skb_frag_unref() we need
1033 * to make one SKB responsible for triggering the recycle path.
1034 * So disable the recycling bit if an SKB is cloned and we have
1035 * additional references to the fragmented part of the SKB.
1036 * Eventually the last SKB will have the recycling bit set and it's
1037 * dataref set to 0, which will trigger the recycling
1039 skb->pp_recycle = 0;
1043 * Free an skbuff by memory without cleaning the state.
1045 static void kfree_skbmem(struct sk_buff *skb)
1047 struct sk_buff_fclones *fclones;
1049 switch (skb->fclone) {
1050 case SKB_FCLONE_UNAVAILABLE:
1051 kmem_cache_free(skbuff_cache, skb);
1054 case SKB_FCLONE_ORIG:
1055 fclones = container_of(skb, struct sk_buff_fclones, skb1);
1057 /* We usually free the clone (TX completion) before original skb
1058 * This test would have no chance to be true for the clone,
1059 * while here, branch prediction will be good.
1061 if (refcount_read(&fclones->fclone_ref) == 1)
1065 default: /* SKB_FCLONE_CLONE */
1066 fclones = container_of(skb, struct sk_buff_fclones, skb2);
1069 if (!refcount_dec_and_test(&fclones->fclone_ref))
1072 kmem_cache_free(skbuff_fclone_cache, fclones);
1075 void skb_release_head_state(struct sk_buff *skb)
1078 if (skb->destructor) {
1079 DEBUG_NET_WARN_ON_ONCE(in_hardirq());
1080 skb->destructor(skb);
1082 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
1083 nf_conntrack_put(skb_nfct(skb));
1088 /* Free everything but the sk_buff shell. */
1089 static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason,
1092 skb_release_head_state(skb);
1093 if (likely(skb->head))
1094 skb_release_data(skb, reason, napi_safe);
1098 * __kfree_skb - private function
1101 * Free an sk_buff. Release anything attached to the buffer.
1102 * Clean the state. This is an internal helper function. Users should
1103 * always call kfree_skb
1106 void __kfree_skb(struct sk_buff *skb)
1108 skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED, false);
1111 EXPORT_SYMBOL(__kfree_skb);
1113 static __always_inline
1114 bool __kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
1116 if (unlikely(!skb_unref(skb)))
1119 DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET ||
1120 u32_get_bits(reason,
1121 SKB_DROP_REASON_SUBSYS_MASK) >=
1122 SKB_DROP_REASON_SUBSYS_NUM);
1124 if (reason == SKB_CONSUMED)
1125 trace_consume_skb(skb, __builtin_return_address(0));
1127 trace_kfree_skb(skb, __builtin_return_address(0), reason);
1132 * kfree_skb_reason - free an sk_buff with special reason
1133 * @skb: buffer to free
1134 * @reason: reason why this skb is dropped
1136 * Drop a reference to the buffer and free it if the usage count has
1137 * hit zero. Meanwhile, pass the drop reason to 'kfree_skb'
1141 kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
1143 if (__kfree_skb_reason(skb, reason))
1146 EXPORT_SYMBOL(kfree_skb_reason);
1148 #define KFREE_SKB_BULK_SIZE 16
1150 struct skb_free_array {
1151 unsigned int skb_count;
1152 void *skb_array[KFREE_SKB_BULK_SIZE];
1155 static void kfree_skb_add_bulk(struct sk_buff *skb,
1156 struct skb_free_array *sa,
1157 enum skb_drop_reason reason)
1159 /* if SKB is a clone, don't handle this case */
1160 if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) {
1165 skb_release_all(skb, reason, false);
1166 sa->skb_array[sa->skb_count++] = skb;
1168 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {
1169 kmem_cache_free_bulk(skbuff_cache, KFREE_SKB_BULK_SIZE,
1176 kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason)
1178 struct skb_free_array sa;
1183 struct sk_buff *next = segs->next;
1185 if (__kfree_skb_reason(segs, reason)) {
1186 skb_poison_list(segs);
1187 kfree_skb_add_bulk(segs, &sa, reason);
1194 kmem_cache_free_bulk(skbuff_cache, sa.skb_count, sa.skb_array);
1196 EXPORT_SYMBOL(kfree_skb_list_reason);
1198 /* Dump skb information and contents.
1200 * Must only be called from net_ratelimit()-ed paths.
1202 * Dumps whole packets if full_pkt, only headers otherwise.
1204 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
1206 struct skb_shared_info *sh = skb_shinfo(skb);
1207 struct net_device *dev = skb->dev;
1208 struct sock *sk = skb->sk;
1209 struct sk_buff *list_skb;
1210 bool has_mac, has_trans;
1211 int headroom, tailroom;
1212 int i, len, seg_len;
1217 len = min_t(int, skb->len, MAX_HEADER + 128);
1219 headroom = skb_headroom(skb);
1220 tailroom = skb_tailroom(skb);
1222 has_mac = skb_mac_header_was_set(skb);
1223 has_trans = skb_transport_header_was_set(skb);
1225 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
1226 "mac=(%d,%d) net=(%d,%d) trans=%d\n"
1227 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
1228 "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
1229 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n",
1230 level, skb->len, headroom, skb_headlen(skb), tailroom,
1231 has_mac ? skb->mac_header : -1,
1232 has_mac ? skb_mac_header_len(skb) : -1,
1233 skb->network_header,
1234 has_trans ? skb_network_header_len(skb) : -1,
1235 has_trans ? skb->transport_header : -1,
1236 sh->tx_flags, sh->nr_frags,
1237 sh->gso_size, sh->gso_type, sh->gso_segs,
1238 skb->csum, skb->ip_summed, skb->csum_complete_sw,
1239 skb->csum_valid, skb->csum_level,
1240 skb->hash, skb->sw_hash, skb->l4_hash,
1241 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
1244 printk("%sdev name=%s feat=%pNF\n",
1245 level, dev->name, &dev->features);
1247 printk("%ssk family=%hu type=%u proto=%u\n",
1248 level, sk->sk_family, sk->sk_type, sk->sk_protocol);
1250 if (full_pkt && headroom)
1251 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
1252 16, 1, skb->head, headroom, false);
1254 seg_len = min_t(int, skb_headlen(skb), len);
1256 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET,
1257 16, 1, skb->data, seg_len, false);
1260 if (full_pkt && tailroom)
1261 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
1262 16, 1, skb_tail_pointer(skb), tailroom, false);
1264 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
1265 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1266 u32 p_off, p_len, copied;
1270 skb_frag_foreach_page(frag, skb_frag_off(frag),
1271 skb_frag_size(frag), p, p_off, p_len,
1273 seg_len = min_t(int, p_len, len);
1274 vaddr = kmap_atomic(p);
1275 print_hex_dump(level, "skb frag: ",
1277 16, 1, vaddr + p_off, seg_len, false);
1278 kunmap_atomic(vaddr);
1285 if (full_pkt && skb_has_frag_list(skb)) {
1286 printk("skb fraglist:\n");
1287 skb_walk_frags(skb, list_skb)
1288 skb_dump(level, list_skb, true);
1291 EXPORT_SYMBOL(skb_dump);
1294 * skb_tx_error - report an sk_buff xmit error
1295 * @skb: buffer that triggered an error
1297 * Report xmit error if a device callback is tracking this skb.
1298 * skb must be freed afterwards.
1300 void skb_tx_error(struct sk_buff *skb)
1303 skb_zcopy_downgrade_managed(skb);
1304 skb_zcopy_clear(skb, true);
1307 EXPORT_SYMBOL(skb_tx_error);
1309 #ifdef CONFIG_TRACEPOINTS
1311 * consume_skb - free an skbuff
1312 * @skb: buffer to free
1314 * Drop a ref to the buffer and free it if the usage count has hit zero
1315 * Functions identically to kfree_skb, but kfree_skb assumes that the frame
1316 * is being dropped after a failure and notes that
1318 void consume_skb(struct sk_buff *skb)
1320 if (!skb_unref(skb))
1323 trace_consume_skb(skb, __builtin_return_address(0));
1326 EXPORT_SYMBOL(consume_skb);
1330 * __consume_stateless_skb - free an skbuff, assuming it is stateless
1331 * @skb: buffer to free
1333 * Alike consume_skb(), but this variant assumes that this is the last
1334 * skb reference and all the head states have been already dropped
1336 void __consume_stateless_skb(struct sk_buff *skb)
1338 trace_consume_skb(skb, __builtin_return_address(0));
1339 skb_release_data(skb, SKB_CONSUMED, false);
1343 static void napi_skb_cache_put(struct sk_buff *skb)
1345 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
1348 if (!kasan_mempool_poison_object(skb))
1351 nc->skb_cache[nc->skb_count++] = skb;
1353 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
1354 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)
1355 kasan_mempool_unpoison_object(nc->skb_cache[i],
1356 kmem_cache_size(skbuff_cache));
1358 kmem_cache_free_bulk(skbuff_cache, NAPI_SKB_CACHE_HALF,
1359 nc->skb_cache + NAPI_SKB_CACHE_HALF);
1360 nc->skb_count = NAPI_SKB_CACHE_HALF;
1364 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason)
1366 skb_release_all(skb, reason, true);
1367 napi_skb_cache_put(skb);
1370 void napi_skb_free_stolen_head(struct sk_buff *skb)
1372 if (unlikely(skb->slow_gro)) {
1379 napi_skb_cache_put(skb);
1382 void napi_consume_skb(struct sk_buff *skb, int budget)
1384 /* Zero budget indicate non-NAPI context called us, like netpoll */
1385 if (unlikely(!budget)) {
1386 dev_consume_skb_any(skb);
1390 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
1392 if (!skb_unref(skb))
1395 /* if reaching here SKB is ready to free */
1396 trace_consume_skb(skb, __builtin_return_address(0));
1398 /* if SKB is a clone, don't handle this case */
1399 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
1404 skb_release_all(skb, SKB_CONSUMED, !!budget);
1405 napi_skb_cache_put(skb);
1407 EXPORT_SYMBOL(napi_consume_skb);
1409 /* Make sure a field is contained by headers group */
1410 #define CHECK_SKB_FIELD(field) \
1411 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \
1412 offsetof(struct sk_buff, headers.field)); \
1414 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1416 new->tstamp = old->tstamp;
1417 /* We do not copy old->sk */
1418 new->dev = old->dev;
1419 memcpy(new->cb, old->cb, sizeof(old->cb));
1420 skb_dst_copy(new, old);
1421 __skb_ext_copy(new, old);
1422 __nf_copy(new, old, false);
1424 /* Note : this field could be in the headers group.
1425 * It is not yet because we do not want to have a 16 bit hole
1427 new->queue_mapping = old->queue_mapping;
1429 memcpy(&new->headers, &old->headers, sizeof(new->headers));
1430 CHECK_SKB_FIELD(protocol);
1431 CHECK_SKB_FIELD(csum);
1432 CHECK_SKB_FIELD(hash);
1433 CHECK_SKB_FIELD(priority);
1434 CHECK_SKB_FIELD(skb_iif);
1435 CHECK_SKB_FIELD(vlan_proto);
1436 CHECK_SKB_FIELD(vlan_tci);
1437 CHECK_SKB_FIELD(transport_header);
1438 CHECK_SKB_FIELD(network_header);
1439 CHECK_SKB_FIELD(mac_header);
1440 CHECK_SKB_FIELD(inner_protocol);
1441 CHECK_SKB_FIELD(inner_transport_header);
1442 CHECK_SKB_FIELD(inner_network_header);
1443 CHECK_SKB_FIELD(inner_mac_header);
1444 CHECK_SKB_FIELD(mark);
1445 #ifdef CONFIG_NETWORK_SECMARK
1446 CHECK_SKB_FIELD(secmark);
1448 #ifdef CONFIG_NET_RX_BUSY_POLL
1449 CHECK_SKB_FIELD(napi_id);
1451 CHECK_SKB_FIELD(alloc_cpu);
1453 CHECK_SKB_FIELD(sender_cpu);
1455 #ifdef CONFIG_NET_SCHED
1456 CHECK_SKB_FIELD(tc_index);
1462 * You should not add any new code to this function. Add it to
1463 * __copy_skb_header above instead.
1465 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
1467 #define C(x) n->x = skb->x
1469 n->next = n->prev = NULL;
1471 __copy_skb_header(n, skb);
1476 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
1482 n->destructor = NULL;
1489 refcount_set(&n->users, 1);
1491 atomic_inc(&(skb_shinfo(skb)->dataref));
1499 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
1500 * @first: first sk_buff of the msg
1502 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
1506 n = alloc_skb(0, GFP_ATOMIC);
1510 n->len = first->len;
1511 n->data_len = first->len;
1512 n->truesize = first->truesize;
1514 skb_shinfo(n)->frag_list = first;
1516 __copy_skb_header(n, first);
1517 n->destructor = NULL;
1521 EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
1524 * skb_morph - morph one skb into another
1525 * @dst: the skb to receive the contents
1526 * @src: the skb to supply the contents
1528 * This is identical to skb_clone except that the target skb is
1529 * supplied by the user.
1531 * The target skb is returned upon exit.
1533 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
1535 skb_release_all(dst, SKB_CONSUMED, false);
1536 return __skb_clone(dst, src);
1538 EXPORT_SYMBOL_GPL(skb_morph);
1540 int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
1542 unsigned long max_pg, num_pg, new_pg, old_pg, rlim;
1543 struct user_struct *user;
1545 if (capable(CAP_IPC_LOCK) || !size)
1548 rlim = rlimit(RLIMIT_MEMLOCK);
1549 if (rlim == RLIM_INFINITY)
1552 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */
1553 max_pg = rlim >> PAGE_SHIFT;
1554 user = mmp->user ? : current_user();
1556 old_pg = atomic_long_read(&user->locked_vm);
1558 new_pg = old_pg + num_pg;
1559 if (new_pg > max_pg)
1561 } while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg));
1564 mmp->user = get_uid(user);
1565 mmp->num_pg = num_pg;
1567 mmp->num_pg += num_pg;
1572 EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
1574 void mm_unaccount_pinned_pages(struct mmpin *mmp)
1577 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
1578 free_uid(mmp->user);
1581 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
1583 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
1585 struct ubuf_info_msgzc *uarg;
1586 struct sk_buff *skb;
1588 WARN_ON_ONCE(!in_task());
1590 skb = sock_omalloc(sk, 0, GFP_KERNEL);
1594 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1595 uarg = (void *)skb->cb;
1596 uarg->mmp.user = NULL;
1598 if (mm_account_pinned_pages(&uarg->mmp, size)) {
1603 uarg->ubuf.callback = msg_zerocopy_callback;
1604 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
1606 uarg->bytelen = size;
1608 uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
1609 refcount_set(&uarg->ubuf.refcnt, 1);
1615 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg)
1617 return container_of((void *)uarg, struct sk_buff, cb);
1620 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
1621 struct ubuf_info *uarg)
1624 struct ubuf_info_msgzc *uarg_zc;
1625 const u32 byte_limit = 1 << 19; /* limit to a few TSO */
1628 /* there might be non MSG_ZEROCOPY users */
1629 if (uarg->callback != msg_zerocopy_callback)
1632 /* realloc only when socket is locked (TCP, UDP cork),
1633 * so uarg->len and sk_zckey access is serialized
1635 if (!sock_owned_by_user(sk)) {
1640 uarg_zc = uarg_to_msgzc(uarg);
1641 bytelen = uarg_zc->bytelen + size;
1642 if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) {
1643 /* TCP can create new skb to attach new uarg */
1644 if (sk->sk_type == SOCK_STREAM)
1649 next = (u32)atomic_read(&sk->sk_zckey);
1650 if ((u32)(uarg_zc->id + uarg_zc->len) == next) {
1651 if (mm_account_pinned_pages(&uarg_zc->mmp, size))
1654 uarg_zc->bytelen = bytelen;
1655 atomic_set(&sk->sk_zckey, ++next);
1657 /* no extra ref when appending to datagram (MSG_MORE) */
1658 if (sk->sk_type == SOCK_STREAM)
1659 net_zcopy_get(uarg);
1666 return msg_zerocopy_alloc(sk, size);
1668 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc);
1670 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1672 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1676 old_lo = serr->ee.ee_info;
1677 old_hi = serr->ee.ee_data;
1678 sum_len = old_hi - old_lo + 1ULL + len;
1680 if (sum_len >= (1ULL << 32))
1683 if (lo != old_hi + 1)
1686 serr->ee.ee_data += len;
1690 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
1692 struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1693 struct sock_exterr_skb *serr;
1694 struct sock *sk = skb->sk;
1695 struct sk_buff_head *q;
1696 unsigned long flags;
1701 mm_unaccount_pinned_pages(&uarg->mmp);
1703 /* if !len, there was only 1 call, and it was aborted
1704 * so do not queue a completion notification
1706 if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1711 hi = uarg->id + len - 1;
1712 is_zerocopy = uarg->zerocopy;
1714 serr = SKB_EXT_ERR(skb);
1715 memset(serr, 0, sizeof(*serr));
1716 serr->ee.ee_errno = 0;
1717 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1718 serr->ee.ee_data = hi;
1719 serr->ee.ee_info = lo;
1721 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1723 q = &sk->sk_error_queue;
1724 spin_lock_irqsave(&q->lock, flags);
1725 tail = skb_peek_tail(q);
1726 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1727 !skb_zerocopy_notify_extend(tail, lo, len)) {
1728 __skb_queue_tail(q, skb);
1731 spin_unlock_irqrestore(&q->lock, flags);
1733 sk_error_report(sk);
1740 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
1743 struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
1745 uarg_zc->zerocopy = uarg_zc->zerocopy & success;
1747 if (refcount_dec_and_test(&uarg->refcnt))
1748 __msg_zerocopy_callback(uarg_zc);
1750 EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
1752 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1754 struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk;
1756 atomic_dec(&sk->sk_zckey);
1757 uarg_to_msgzc(uarg)->len--;
1760 msg_zerocopy_callback(NULL, uarg, true);
1762 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
1764 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1765 struct msghdr *msg, int len,
1766 struct ubuf_info *uarg)
1768 struct ubuf_info *orig_uarg = skb_zcopy(skb);
1769 int err, orig_len = skb->len;
1771 /* An skb can only point to one uarg. This edge case happens when
1772 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1774 if (orig_uarg && uarg != orig_uarg)
1777 err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len);
1778 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1779 struct sock *save_sk = skb->sk;
1781 /* Streams do not free skb on error. Reset to prev state. */
1782 iov_iter_revert(&msg->msg_iter, skb->len - orig_len);
1784 ___pskb_trim(skb, orig_len);
1789 skb_zcopy_set(skb, uarg, NULL);
1790 return skb->len - orig_len;
1792 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1794 void __skb_zcopy_downgrade_managed(struct sk_buff *skb)
1798 skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS;
1799 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1800 skb_frag_ref(skb, i);
1802 EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed);
1804 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1807 if (skb_zcopy(orig)) {
1808 if (skb_zcopy(nskb)) {
1809 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1814 if (skb_uarg(nskb) == skb_uarg(orig))
1816 if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1819 skb_zcopy_set(nskb, skb_uarg(orig), NULL);
1825 * skb_copy_ubufs - copy userspace skb frags buffers to kernel
1826 * @skb: the skb to modify
1827 * @gfp_mask: allocation priority
1829 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE.
1830 * It will copy all frags into kernel and drop the reference
1831 * to userspace pages.
1833 * If this function is called from an interrupt gfp_mask() must be
1836 * Returns 0 on success or a negative error code on failure
1837 * to allocate kernel memory to copy to.
1839 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1841 int num_frags = skb_shinfo(skb)->nr_frags;
1842 struct page *page, *head = NULL;
1843 int i, order, psize, new_frags;
1846 if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1852 /* We might have to allocate high order pages, so compute what minimum
1853 * page order is needed.
1856 while ((PAGE_SIZE << order) * MAX_SKB_FRAGS < __skb_pagelen(skb))
1858 psize = (PAGE_SIZE << order);
1860 new_frags = (__skb_pagelen(skb) + psize - 1) >> (PAGE_SHIFT + order);
1861 for (i = 0; i < new_frags; i++) {
1862 page = alloc_pages(gfp_mask | __GFP_COMP, order);
1865 struct page *next = (struct page *)page_private(head);
1871 set_page_private(page, (unsigned long)head);
1877 for (i = 0; i < num_frags; i++) {
1878 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1879 u32 p_off, p_len, copied;
1883 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
1884 p, p_off, p_len, copied) {
1886 vaddr = kmap_atomic(p);
1888 while (done < p_len) {
1889 if (d_off == psize) {
1891 page = (struct page *)page_private(page);
1893 copy = min_t(u32, psize - d_off, p_len - done);
1894 memcpy(page_address(page) + d_off,
1895 vaddr + p_off + done, copy);
1899 kunmap_atomic(vaddr);
1903 /* skb frags release userspace buffers */
1904 for (i = 0; i < num_frags; i++)
1905 skb_frag_unref(skb, i);
1907 /* skb frags point to kernel buffers */
1908 for (i = 0; i < new_frags - 1; i++) {
1909 __skb_fill_page_desc(skb, i, head, 0, psize);
1910 head = (struct page *)page_private(head);
1912 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
1913 skb_shinfo(skb)->nr_frags = new_frags;
1916 skb_zcopy_clear(skb, false);
1919 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
1922 * skb_clone - duplicate an sk_buff
1923 * @skb: buffer to clone
1924 * @gfp_mask: allocation priority
1926 * Duplicate an &sk_buff. The new one is not owned by a socket. Both
1927 * copies share the same packet data but not structure. The new
1928 * buffer has a reference count of 1. If the allocation fails the
1929 * function returns %NULL otherwise the new buffer is returned.
1931 * If this function is called from an interrupt gfp_mask() must be
1935 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
1937 struct sk_buff_fclones *fclones = container_of(skb,
1938 struct sk_buff_fclones,
1942 if (skb_orphan_frags(skb, gfp_mask))
1945 if (skb->fclone == SKB_FCLONE_ORIG &&
1946 refcount_read(&fclones->fclone_ref) == 1) {
1948 refcount_set(&fclones->fclone_ref, 2);
1949 n->fclone = SKB_FCLONE_CLONE;
1951 if (skb_pfmemalloc(skb))
1952 gfp_mask |= __GFP_MEMALLOC;
1954 n = kmem_cache_alloc(skbuff_cache, gfp_mask);
1958 n->fclone = SKB_FCLONE_UNAVAILABLE;
1961 return __skb_clone(n, skb);
1963 EXPORT_SYMBOL(skb_clone);
1965 void skb_headers_offset_update(struct sk_buff *skb, int off)
1967 /* Only adjust this if it actually is csum_start rather than csum */
1968 if (skb->ip_summed == CHECKSUM_PARTIAL)
1969 skb->csum_start += off;
1970 /* {transport,network,mac}_header and tail are relative to skb->head */
1971 skb->transport_header += off;
1972 skb->network_header += off;
1973 if (skb_mac_header_was_set(skb))
1974 skb->mac_header += off;
1975 skb->inner_transport_header += off;
1976 skb->inner_network_header += off;
1977 skb->inner_mac_header += off;
1979 EXPORT_SYMBOL(skb_headers_offset_update);
1981 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
1983 __copy_skb_header(new, old);
1985 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
1986 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
1987 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
1989 EXPORT_SYMBOL(skb_copy_header);
1991 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
1993 if (skb_pfmemalloc(skb))
1994 return SKB_ALLOC_RX;
1999 * skb_copy - create private copy of an sk_buff
2000 * @skb: buffer to copy
2001 * @gfp_mask: allocation priority
2003 * Make a copy of both an &sk_buff and its data. This is used when the
2004 * caller wishes to modify the data and needs a private copy of the
2005 * data to alter. Returns %NULL on failure or the pointer to the buffer
2006 * on success. The returned buffer has a reference count of 1.
2008 * As by-product this function converts non-linear &sk_buff to linear
2009 * one, so that &sk_buff becomes completely private and caller is allowed
2010 * to modify all the data of returned buffer. This means that this
2011 * function is not recommended for use in circumstances when only
2012 * header is going to be modified. Use pskb_copy() instead.
2015 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
2017 int headerlen = skb_headroom(skb);
2018 unsigned int size = skb_end_offset(skb) + skb->data_len;
2019 struct sk_buff *n = __alloc_skb(size, gfp_mask,
2020 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
2025 /* Set the data pointer */
2026 skb_reserve(n, headerlen);
2027 /* Set the tail pointer and length */
2028 skb_put(n, skb->len);
2030 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
2032 skb_copy_header(n, skb);
2035 EXPORT_SYMBOL(skb_copy);
2038 * __pskb_copy_fclone - create copy of an sk_buff with private head.
2039 * @skb: buffer to copy
2040 * @headroom: headroom of new skb
2041 * @gfp_mask: allocation priority
2042 * @fclone: if true allocate the copy of the skb from the fclone
2043 * cache instead of the head cache; it is recommended to set this
2044 * to true for the cases where the copy will likely be cloned
2046 * Make a copy of both an &sk_buff and part of its data, located
2047 * in header. Fragmented data remain shared. This is used when
2048 * the caller wishes to modify only header of &sk_buff and needs
2049 * private copy of the header to alter. Returns %NULL on failure
2050 * or the pointer to the buffer on success.
2051 * The returned buffer has a reference count of 1.
2054 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
2055 gfp_t gfp_mask, bool fclone)
2057 unsigned int size = skb_headlen(skb) + headroom;
2058 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
2059 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
2064 /* Set the data pointer */
2065 skb_reserve(n, headroom);
2066 /* Set the tail pointer and length */
2067 skb_put(n, skb_headlen(skb));
2068 /* Copy the bytes */
2069 skb_copy_from_linear_data(skb, n->data, n->len);
2071 n->truesize += skb->data_len;
2072 n->data_len = skb->data_len;
2075 if (skb_shinfo(skb)->nr_frags) {
2078 if (skb_orphan_frags(skb, gfp_mask) ||
2079 skb_zerocopy_clone(n, skb, gfp_mask)) {
2084 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2085 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
2086 skb_frag_ref(skb, i);
2088 skb_shinfo(n)->nr_frags = i;
2091 if (skb_has_frag_list(skb)) {
2092 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
2093 skb_clone_fraglist(n);
2096 skb_copy_header(n, skb);
2100 EXPORT_SYMBOL(__pskb_copy_fclone);
2103 * pskb_expand_head - reallocate header of &sk_buff
2104 * @skb: buffer to reallocate
2105 * @nhead: room to add at head
2106 * @ntail: room to add at tail
2107 * @gfp_mask: allocation priority
2109 * Expands (or creates identical copy, if @nhead and @ntail are zero)
2110 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
2111 * reference count of 1. Returns zero in the case of success or error,
2112 * if expansion failed. In the last case, &sk_buff is not changed.
2114 * All the pointers pointing into skb header may change and must be
2115 * reloaded after call to this function.
2118 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
2121 unsigned int osize = skb_end_offset(skb);
2122 unsigned int size = osize + nhead + ntail;
2129 BUG_ON(skb_shared(skb));
2131 skb_zcopy_downgrade_managed(skb);
2133 if (skb_pfmemalloc(skb))
2134 gfp_mask |= __GFP_MEMALLOC;
2136 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
2139 size = SKB_WITH_OVERHEAD(size);
2141 /* Copy only real data... and, alas, header. This should be
2142 * optimized for the cases when header is void.
2144 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
2146 memcpy((struct skb_shared_info *)(data + size),
2148 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
2151 * if shinfo is shared we must drop the old head gracefully, but if it
2152 * is not we can just drop the old head and let the existing refcount
2153 * be since all we did is relocate the values
2155 if (skb_cloned(skb)) {
2156 if (skb_orphan_frags(skb, gfp_mask))
2159 refcount_inc(&skb_uarg(skb)->refcnt);
2160 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2161 skb_frag_ref(skb, i);
2163 if (skb_has_frag_list(skb))
2164 skb_clone_fraglist(skb);
2166 skb_release_data(skb, SKB_CONSUMED, false);
2168 skb_free_head(skb, false);
2170 off = (data + nhead) - skb->head;
2176 skb_set_end_offset(skb, size);
2177 #ifdef NET_SKBUFF_DATA_USES_OFFSET
2181 skb_headers_offset_update(skb, nhead);
2185 atomic_set(&skb_shinfo(skb)->dataref, 1);
2187 skb_metadata_clear(skb);
2189 /* It is not generally safe to change skb->truesize.
2190 * For the moment, we really care of rx path, or
2191 * when skb is orphaned (not attached to a socket).
2193 if (!skb->sk || skb->destructor == sock_edemux)
2194 skb->truesize += size - osize;
2199 skb_kfree_head(data, size);
2203 EXPORT_SYMBOL(pskb_expand_head);
2205 /* Make private copy of skb with writable head and some headroom */
2207 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
2209 struct sk_buff *skb2;
2210 int delta = headroom - skb_headroom(skb);
2213 skb2 = pskb_copy(skb, GFP_ATOMIC);
2215 skb2 = skb_clone(skb, GFP_ATOMIC);
2216 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
2224 EXPORT_SYMBOL(skb_realloc_headroom);
2226 /* Note: We plan to rework this in linux-6.4 */
2227 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
2229 unsigned int saved_end_offset, saved_truesize;
2230 struct skb_shared_info *shinfo;
2233 saved_end_offset = skb_end_offset(skb);
2234 saved_truesize = skb->truesize;
2236 res = pskb_expand_head(skb, 0, 0, pri);
2240 skb->truesize = saved_truesize;
2242 if (likely(skb_end_offset(skb) == saved_end_offset))
2245 /* We can not change skb->end if the original or new value
2246 * is SKB_SMALL_HEAD_HEADROOM, as it might break skb_kfree_head().
2248 if (saved_end_offset == SKB_SMALL_HEAD_HEADROOM ||
2249 skb_end_offset(skb) == SKB_SMALL_HEAD_HEADROOM) {
2250 /* We think this path should not be taken.
2251 * Add a temporary trace to warn us just in case.
2253 pr_err_once("__skb_unclone_keeptruesize() skb_end_offset() %u -> %u\n",
2254 saved_end_offset, skb_end_offset(skb));
2259 shinfo = skb_shinfo(skb);
2261 /* We are about to change back skb->end,
2262 * we need to move skb_shinfo() to its new location.
2264 memmove(skb->head + saved_end_offset,
2266 offsetof(struct skb_shared_info, frags[shinfo->nr_frags]));
2268 skb_set_end_offset(skb, saved_end_offset);
2274 * skb_expand_head - reallocate header of &sk_buff
2275 * @skb: buffer to reallocate
2276 * @headroom: needed headroom
2278 * Unlike skb_realloc_headroom, this one does not allocate a new skb
2279 * if possible; copies skb->sk to new skb as needed
2280 * and frees original skb in case of failures.
2282 * It expect increased headroom and generates warning otherwise.
2285 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom)
2287 int delta = headroom - skb_headroom(skb);
2288 int osize = skb_end_offset(skb);
2289 struct sock *sk = skb->sk;
2291 if (WARN_ONCE(delta <= 0,
2292 "%s is expecting an increase in the headroom", __func__))
2295 delta = SKB_DATA_ALIGN(delta);
2296 /* pskb_expand_head() might crash, if skb is shared. */
2297 if (skb_shared(skb) || !is_skb_wmem(skb)) {
2298 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2300 if (unlikely(!nskb))
2304 skb_set_owner_w(nskb, sk);
2308 if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC))
2311 if (sk && is_skb_wmem(skb)) {
2312 delta = skb_end_offset(skb) - osize;
2313 refcount_add(delta, &sk->sk_wmem_alloc);
2314 skb->truesize += delta;
2322 EXPORT_SYMBOL(skb_expand_head);
2325 * skb_copy_expand - copy and expand sk_buff
2326 * @skb: buffer to copy
2327 * @newheadroom: new free bytes at head
2328 * @newtailroom: new free bytes at tail
2329 * @gfp_mask: allocation priority
2331 * Make a copy of both an &sk_buff and its data and while doing so
2332 * allocate additional space.
2334 * This is used when the caller wishes to modify the data and needs a
2335 * private copy of the data to alter as well as more space for new fields.
2336 * Returns %NULL on failure or the pointer to the buffer
2337 * on success. The returned buffer has a reference count of 1.
2339 * You must pass %GFP_ATOMIC as the allocation priority if this function
2340 * is called from an interrupt.
2342 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
2343 int newheadroom, int newtailroom,
2347 * Allocate the copy buffer
2349 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
2350 gfp_mask, skb_alloc_rx_flag(skb),
2352 int oldheadroom = skb_headroom(skb);
2353 int head_copy_len, head_copy_off;
2358 skb_reserve(n, newheadroom);
2360 /* Set the tail pointer and length */
2361 skb_put(n, skb->len);
2363 head_copy_len = oldheadroom;
2365 if (newheadroom <= head_copy_len)
2366 head_copy_len = newheadroom;
2368 head_copy_off = newheadroom - head_copy_len;
2370 /* Copy the linear header and data. */
2371 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
2372 skb->len + head_copy_len));
2374 skb_copy_header(n, skb);
2376 skb_headers_offset_update(n, newheadroom - oldheadroom);
2380 EXPORT_SYMBOL(skb_copy_expand);
2383 * __skb_pad - zero pad the tail of an skb
2384 * @skb: buffer to pad
2385 * @pad: space to pad
2386 * @free_on_error: free buffer on error
2388 * Ensure that a buffer is followed by a padding area that is zero
2389 * filled. Used by network drivers which may DMA or transfer data
2390 * beyond the buffer end onto the wire.
2392 * May return error in out of memory cases. The skb is freed on error
2393 * if @free_on_error is true.
2396 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
2401 /* If the skbuff is non linear tailroom is always zero.. */
2402 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
2403 memset(skb->data+skb->len, 0, pad);
2407 ntail = skb->data_len + pad - (skb->end - skb->tail);
2408 if (likely(skb_cloned(skb) || ntail > 0)) {
2409 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
2414 /* FIXME: The use of this function with non-linear skb's really needs
2417 err = skb_linearize(skb);
2421 memset(skb->data + skb->len, 0, pad);
2429 EXPORT_SYMBOL(__skb_pad);
2432 * pskb_put - add data to the tail of a potentially fragmented buffer
2433 * @skb: start of the buffer to use
2434 * @tail: tail fragment of the buffer to use
2435 * @len: amount of data to add
2437 * This function extends the used data area of the potentially
2438 * fragmented buffer. @tail must be the last fragment of @skb -- or
2439 * @skb itself. If this would exceed the total buffer size the kernel
2440 * will panic. A pointer to the first byte of the extra data is
2444 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
2447 skb->data_len += len;
2450 return skb_put(tail, len);
2452 EXPORT_SYMBOL_GPL(pskb_put);
2455 * skb_put - add data to a buffer
2456 * @skb: buffer to use
2457 * @len: amount of data to add
2459 * This function extends the used data area of the buffer. If this would
2460 * exceed the total buffer size the kernel will panic. A pointer to the
2461 * first byte of the extra data is returned.
2463 void *skb_put(struct sk_buff *skb, unsigned int len)
2465 void *tmp = skb_tail_pointer(skb);
2466 SKB_LINEAR_ASSERT(skb);
2469 if (unlikely(skb->tail > skb->end))
2470 skb_over_panic(skb, len, __builtin_return_address(0));
2473 EXPORT_SYMBOL(skb_put);
2476 * skb_push - add data to the start of a buffer
2477 * @skb: buffer to use
2478 * @len: amount of data to add
2480 * This function extends the used data area of the buffer at the buffer
2481 * start. If this would exceed the total buffer headroom the kernel will
2482 * panic. A pointer to the first byte of the extra data is returned.
2484 void *skb_push(struct sk_buff *skb, unsigned int len)
2488 if (unlikely(skb->data < skb->head))
2489 skb_under_panic(skb, len, __builtin_return_address(0));
2492 EXPORT_SYMBOL(skb_push);
2495 * skb_pull - remove data from the start of a buffer
2496 * @skb: buffer to use
2497 * @len: amount of data to remove
2499 * This function removes data from the start of a buffer, returning
2500 * the memory to the headroom. A pointer to the next data in the buffer
2501 * is returned. Once the data has been pulled future pushes will overwrite
2504 void *skb_pull(struct sk_buff *skb, unsigned int len)
2506 return skb_pull_inline(skb, len);
2508 EXPORT_SYMBOL(skb_pull);
2511 * skb_pull_data - remove data from the start of a buffer returning its
2512 * original position.
2513 * @skb: buffer to use
2514 * @len: amount of data to remove
2516 * This function removes data from the start of a buffer, returning
2517 * the memory to the headroom. A pointer to the original data in the buffer
2518 * is returned after checking if there is enough data to pull. Once the
2519 * data has been pulled future pushes will overwrite the old data.
2521 void *skb_pull_data(struct sk_buff *skb, size_t len)
2523 void *data = skb->data;
2532 EXPORT_SYMBOL(skb_pull_data);
2535 * skb_trim - remove end from a buffer
2536 * @skb: buffer to alter
2539 * Cut the length of a buffer down by removing data from the tail. If
2540 * the buffer is already under the length specified it is not modified.
2541 * The skb must be linear.
2543 void skb_trim(struct sk_buff *skb, unsigned int len)
2546 __skb_trim(skb, len);
2548 EXPORT_SYMBOL(skb_trim);
2550 /* Trims skb to length len. It can change skb pointers.
2553 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
2555 struct sk_buff **fragp;
2556 struct sk_buff *frag;
2557 int offset = skb_headlen(skb);
2558 int nfrags = skb_shinfo(skb)->nr_frags;
2562 if (skb_cloned(skb) &&
2563 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
2570 for (; i < nfrags; i++) {
2571 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2578 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
2581 skb_shinfo(skb)->nr_frags = i;
2583 for (; i < nfrags; i++)
2584 skb_frag_unref(skb, i);
2586 if (skb_has_frag_list(skb))
2587 skb_drop_fraglist(skb);
2591 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
2592 fragp = &frag->next) {
2593 int end = offset + frag->len;
2595 if (skb_shared(frag)) {
2596 struct sk_buff *nfrag;
2598 nfrag = skb_clone(frag, GFP_ATOMIC);
2599 if (unlikely(!nfrag))
2602 nfrag->next = frag->next;
2614 unlikely((err = pskb_trim(frag, len - offset))))
2618 skb_drop_list(&frag->next);
2623 if (len > skb_headlen(skb)) {
2624 skb->data_len -= skb->len - len;
2629 skb_set_tail_pointer(skb, len);
2632 if (!skb->sk || skb->destructor == sock_edemux)
2636 EXPORT_SYMBOL(___pskb_trim);
2638 /* Note : use pskb_trim_rcsum() instead of calling this directly
2640 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
2642 if (skb->ip_summed == CHECKSUM_COMPLETE) {
2643 int delta = skb->len - len;
2645 skb->csum = csum_block_sub(skb->csum,
2646 skb_checksum(skb, len, delta, 0),
2648 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
2649 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
2650 int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
2652 if (offset + sizeof(__sum16) > hdlen)
2655 return __pskb_trim(skb, len);
2657 EXPORT_SYMBOL(pskb_trim_rcsum_slow);
2660 * __pskb_pull_tail - advance tail of skb header
2661 * @skb: buffer to reallocate
2662 * @delta: number of bytes to advance tail
2664 * The function makes a sense only on a fragmented &sk_buff,
2665 * it expands header moving its tail forward and copying necessary
2666 * data from fragmented part.
2668 * &sk_buff MUST have reference count of 1.
2670 * Returns %NULL (and &sk_buff does not change) if pull failed
2671 * or value of new tail of skb in the case of success.
2673 * All the pointers pointing into skb header may change and must be
2674 * reloaded after call to this function.
2677 /* Moves tail of skb head forward, copying data from fragmented part,
2678 * when it is necessary.
2679 * 1. It may fail due to malloc failure.
2680 * 2. It may change skb pointers.
2682 * It is pretty complicated. Luckily, it is called only in exceptional cases.
2684 void *__pskb_pull_tail(struct sk_buff *skb, int delta)
2686 /* If skb has not enough free space at tail, get new one
2687 * plus 128 bytes for future expansions. If we have enough
2688 * room at tail, reallocate without expansion only if skb is cloned.
2690 int i, k, eat = (skb->tail + delta) - skb->end;
2692 if (eat > 0 || skb_cloned(skb)) {
2693 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
2698 BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
2699 skb_tail_pointer(skb), delta));
2701 /* Optimization: no fragments, no reasons to preestimate
2702 * size of pulled pages. Superb.
2704 if (!skb_has_frag_list(skb))
2707 /* Estimate size of pulled pages. */
2709 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2710 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2717 /* If we need update frag list, we are in troubles.
2718 * Certainly, it is possible to add an offset to skb data,
2719 * but taking into account that pulling is expected to
2720 * be very rare operation, it is worth to fight against
2721 * further bloating skb head and crucify ourselves here instead.
2722 * Pure masohism, indeed. 8)8)
2725 struct sk_buff *list = skb_shinfo(skb)->frag_list;
2726 struct sk_buff *clone = NULL;
2727 struct sk_buff *insp = NULL;
2730 if (list->len <= eat) {
2731 /* Eaten as whole. */
2736 /* Eaten partially. */
2737 if (skb_is_gso(skb) && !list->head_frag &&
2739 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2741 if (skb_shared(list)) {
2742 /* Sucks! We need to fork list. :-( */
2743 clone = skb_clone(list, GFP_ATOMIC);
2749 /* This may be pulled without
2753 if (!pskb_pull(list, eat)) {
2761 /* Free pulled out fragments. */
2762 while ((list = skb_shinfo(skb)->frag_list) != insp) {
2763 skb_shinfo(skb)->frag_list = list->next;
2766 /* And insert new clone at head. */
2769 skb_shinfo(skb)->frag_list = clone;
2772 /* Success! Now we may commit changes to skb data. */
2777 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2778 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2781 skb_frag_unref(skb, i);
2784 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2786 *frag = skb_shinfo(skb)->frags[i];
2788 skb_frag_off_add(frag, eat);
2789 skb_frag_size_sub(frag, eat);
2797 skb_shinfo(skb)->nr_frags = k;
2801 skb->data_len -= delta;
2804 skb_zcopy_clear(skb, false);
2806 return skb_tail_pointer(skb);
2808 EXPORT_SYMBOL(__pskb_pull_tail);
2811 * skb_copy_bits - copy bits from skb to kernel buffer
2813 * @offset: offset in source
2814 * @to: destination buffer
2815 * @len: number of bytes to copy
2817 * Copy the specified number of bytes from the source skb to the
2818 * destination buffer.
2821 * If its prototype is ever changed,
2822 * check arch/{*}/net/{*}.S files,
2823 * since it is called from BPF assembly code.
2825 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2827 int start = skb_headlen(skb);
2828 struct sk_buff *frag_iter;
2831 if (offset > (int)skb->len - len)
2835 if ((copy = start - offset) > 0) {
2838 skb_copy_from_linear_data_offset(skb, offset, to, copy);
2839 if ((len -= copy) == 0)
2845 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2847 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2849 WARN_ON(start > offset + len);
2851 end = start + skb_frag_size(f);
2852 if ((copy = end - offset) > 0) {
2853 u32 p_off, p_len, copied;
2860 skb_frag_foreach_page(f,
2861 skb_frag_off(f) + offset - start,
2862 copy, p, p_off, p_len, copied) {
2863 vaddr = kmap_atomic(p);
2864 memcpy(to + copied, vaddr + p_off, p_len);
2865 kunmap_atomic(vaddr);
2868 if ((len -= copy) == 0)
2876 skb_walk_frags(skb, frag_iter) {
2879 WARN_ON(start > offset + len);
2881 end = start + frag_iter->len;
2882 if ((copy = end - offset) > 0) {
2885 if (skb_copy_bits(frag_iter, offset - start, to, copy))
2887 if ((len -= copy) == 0)
2901 EXPORT_SYMBOL(skb_copy_bits);
2904 * Callback from splice_to_pipe(), if we need to release some pages
2905 * at the end of the spd in case we error'ed out in filling the pipe.
2907 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
2909 put_page(spd->pages[i]);
2912 static struct page *linear_to_page(struct page *page, unsigned int *len,
2913 unsigned int *offset,
2916 struct page_frag *pfrag = sk_page_frag(sk);
2918 if (!sk_page_frag_refill(sk, pfrag))
2921 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
2923 memcpy(page_address(pfrag->page) + pfrag->offset,
2924 page_address(page) + *offset, *len);
2925 *offset = pfrag->offset;
2926 pfrag->offset += *len;
2931 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
2933 unsigned int offset)
2935 return spd->nr_pages &&
2936 spd->pages[spd->nr_pages - 1] == page &&
2937 (spd->partial[spd->nr_pages - 1].offset +
2938 spd->partial[spd->nr_pages - 1].len == offset);
2942 * Fill page/offset/length into spd, if it can hold more pages.
2944 static bool spd_fill_page(struct splice_pipe_desc *spd,
2945 struct pipe_inode_info *pipe, struct page *page,
2946 unsigned int *len, unsigned int offset,
2950 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
2954 page = linear_to_page(page, len, &offset, sk);
2958 if (spd_can_coalesce(spd, page, offset)) {
2959 spd->partial[spd->nr_pages - 1].len += *len;
2963 spd->pages[spd->nr_pages] = page;
2964 spd->partial[spd->nr_pages].len = *len;
2965 spd->partial[spd->nr_pages].offset = offset;
2971 static bool __splice_segment(struct page *page, unsigned int poff,
2972 unsigned int plen, unsigned int *off,
2974 struct splice_pipe_desc *spd, bool linear,
2976 struct pipe_inode_info *pipe)
2981 /* skip this segment if already processed */
2987 /* ignore any bits we already processed */
2993 unsigned int flen = min(*len, plen);
2995 if (spd_fill_page(spd, pipe, page, &flen, poff,
3001 } while (*len && plen);
3007 * Map linear and fragment data from the skb to spd. It reports true if the
3008 * pipe is full or if we already spliced the requested length.
3010 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
3011 unsigned int *offset, unsigned int *len,
3012 struct splice_pipe_desc *spd, struct sock *sk)
3015 struct sk_buff *iter;
3017 /* map the linear part :
3018 * If skb->head_frag is set, this 'linear' part is backed by a
3019 * fragment, and if the head is not shared with any clones then
3020 * we can avoid a copy since we own the head portion of this page.
3022 if (__splice_segment(virt_to_page(skb->data),
3023 (unsigned long) skb->data & (PAGE_SIZE - 1),
3026 skb_head_is_locked(skb),
3031 * then map the fragments
3033 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
3034 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
3036 if (__splice_segment(skb_frag_page(f),
3037 skb_frag_off(f), skb_frag_size(f),
3038 offset, len, spd, false, sk, pipe))
3042 skb_walk_frags(skb, iter) {
3043 if (*offset >= iter->len) {
3044 *offset -= iter->len;
3047 /* __skb_splice_bits() only fails if the output has no room
3048 * left, so no point in going over the frag_list for the error
3051 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
3059 * Map data from the skb to a pipe. Should handle both the linear part,
3060 * the fragments, and the frag list.
3062 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
3063 struct pipe_inode_info *pipe, unsigned int tlen,
3066 struct partial_page partial[MAX_SKB_FRAGS];
3067 struct page *pages[MAX_SKB_FRAGS];
3068 struct splice_pipe_desc spd = {
3071 .nr_pages_max = MAX_SKB_FRAGS,
3072 .ops = &nosteal_pipe_buf_ops,
3073 .spd_release = sock_spd_release,
3077 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
3080 ret = splice_to_pipe(pipe, &spd);
3084 EXPORT_SYMBOL_GPL(skb_splice_bits);
3086 static int sendmsg_locked(struct sock *sk, struct msghdr *msg)
3088 struct socket *sock = sk->sk_socket;
3089 size_t size = msg_data_left(msg);
3094 if (!sock->ops->sendmsg_locked)
3095 return sock_no_sendmsg_locked(sk, msg, size);
3097 return sock->ops->sendmsg_locked(sk, msg, size);
3100 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg)
3102 struct socket *sock = sk->sk_socket;
3106 return sock_sendmsg(sock, msg);
3109 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg);
3110 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
3111 int len, sendmsg_func sendmsg)
3113 unsigned int orig_len = len;
3114 struct sk_buff *head = skb;
3115 unsigned short fragidx;
3120 /* Deal with head data */
3121 while (offset < skb_headlen(skb) && len) {
3125 slen = min_t(int, len, skb_headlen(skb) - offset);
3126 kv.iov_base = skb->data + offset;
3128 memset(&msg, 0, sizeof(msg));
3129 msg.msg_flags = MSG_DONTWAIT;
3131 iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen);
3132 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
3133 sendmsg_unlocked, sk, &msg);
3141 /* All the data was skb head? */
3145 /* Make offset relative to start of frags */
3146 offset -= skb_headlen(skb);
3148 /* Find where we are in frag list */
3149 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
3150 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
3152 if (offset < skb_frag_size(frag))
3155 offset -= skb_frag_size(frag);
3158 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
3159 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
3161 slen = min_t(size_t, len, skb_frag_size(frag) - offset);
3164 struct bio_vec bvec;
3165 struct msghdr msg = {
3166 .msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT,
3169 bvec_set_page(&bvec, skb_frag_page(frag), slen,
3170 skb_frag_off(frag) + offset);
3171 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1,
3174 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
3175 sendmsg_unlocked, sk, &msg);
3188 /* Process any frag lists */
3191 if (skb_has_frag_list(skb)) {
3192 skb = skb_shinfo(skb)->frag_list;
3195 } else if (skb->next) {
3202 return orig_len - len;
3205 return orig_len == len ? ret : orig_len - len;
3208 /* Send skb data on a socket. Socket must be locked. */
3209 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
3212 return __skb_send_sock(sk, skb, offset, len, sendmsg_locked);
3214 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
3216 /* Send skb data on a socket. Socket must be unlocked. */
3217 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
3219 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked);
3223 * skb_store_bits - store bits from kernel buffer to skb
3224 * @skb: destination buffer
3225 * @offset: offset in destination
3226 * @from: source buffer
3227 * @len: number of bytes to copy
3229 * Copy the specified number of bytes from the source buffer to the
3230 * destination skb. This function handles all the messy bits of
3231 * traversing fragment lists and such.
3234 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
3236 int start = skb_headlen(skb);
3237 struct sk_buff *frag_iter;
3240 if (offset > (int)skb->len - len)
3243 if ((copy = start - offset) > 0) {
3246 skb_copy_to_linear_data_offset(skb, offset, from, copy);
3247 if ((len -= copy) == 0)
3253 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3254 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3257 WARN_ON(start > offset + len);
3259 end = start + skb_frag_size(frag);
3260 if ((copy = end - offset) > 0) {
3261 u32 p_off, p_len, copied;
3268 skb_frag_foreach_page(frag,
3269 skb_frag_off(frag) + offset - start,
3270 copy, p, p_off, p_len, copied) {
3271 vaddr = kmap_atomic(p);
3272 memcpy(vaddr + p_off, from + copied, p_len);
3273 kunmap_atomic(vaddr);
3276 if ((len -= copy) == 0)
3284 skb_walk_frags(skb, frag_iter) {
3287 WARN_ON(start > offset + len);
3289 end = start + frag_iter->len;
3290 if ((copy = end - offset) > 0) {
3293 if (skb_store_bits(frag_iter, offset - start,
3296 if ((len -= copy) == 0)
3309 EXPORT_SYMBOL(skb_store_bits);
3311 /* Checksum skb data. */
3312 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
3313 __wsum csum, const struct skb_checksum_ops *ops)
3315 int start = skb_headlen(skb);
3316 int i, copy = start - offset;
3317 struct sk_buff *frag_iter;
3320 /* Checksum header. */
3324 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
3325 skb->data + offset, copy, csum);
3326 if ((len -= copy) == 0)
3332 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3334 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3336 WARN_ON(start > offset + len);
3338 end = start + skb_frag_size(frag);
3339 if ((copy = end - offset) > 0) {
3340 u32 p_off, p_len, copied;
3348 skb_frag_foreach_page(frag,
3349 skb_frag_off(frag) + offset - start,
3350 copy, p, p_off, p_len, copied) {
3351 vaddr = kmap_atomic(p);
3352 csum2 = INDIRECT_CALL_1(ops->update,
3354 vaddr + p_off, p_len, 0);
3355 kunmap_atomic(vaddr);
3356 csum = INDIRECT_CALL_1(ops->combine,
3357 csum_block_add_ext, csum,
3369 skb_walk_frags(skb, frag_iter) {
3372 WARN_ON(start > offset + len);
3374 end = start + frag_iter->len;
3375 if ((copy = end - offset) > 0) {
3379 csum2 = __skb_checksum(frag_iter, offset - start,
3381 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
3382 csum, csum2, pos, copy);
3383 if ((len -= copy) == 0)
3394 EXPORT_SYMBOL(__skb_checksum);
3396 __wsum skb_checksum(const struct sk_buff *skb, int offset,
3397 int len, __wsum csum)
3399 const struct skb_checksum_ops ops = {
3400 .update = csum_partial_ext,
3401 .combine = csum_block_add_ext,
3404 return __skb_checksum(skb, offset, len, csum, &ops);
3406 EXPORT_SYMBOL(skb_checksum);
3408 /* Both of above in one bottle. */
3410 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
3413 int start = skb_headlen(skb);
3414 int i, copy = start - offset;
3415 struct sk_buff *frag_iter;
3423 csum = csum_partial_copy_nocheck(skb->data + offset, to,
3425 if ((len -= copy) == 0)
3432 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3435 WARN_ON(start > offset + len);
3437 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3438 if ((copy = end - offset) > 0) {
3439 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3440 u32 p_off, p_len, copied;
3448 skb_frag_foreach_page(frag,
3449 skb_frag_off(frag) + offset - start,
3450 copy, p, p_off, p_len, copied) {
3451 vaddr = kmap_atomic(p);
3452 csum2 = csum_partial_copy_nocheck(vaddr + p_off,
3455 kunmap_atomic(vaddr);
3456 csum = csum_block_add(csum, csum2, pos);
3468 skb_walk_frags(skb, frag_iter) {
3472 WARN_ON(start > offset + len);
3474 end = start + frag_iter->len;
3475 if ((copy = end - offset) > 0) {
3478 csum2 = skb_copy_and_csum_bits(frag_iter,
3481 csum = csum_block_add(csum, csum2, pos);
3482 if ((len -= copy) == 0)
3493 EXPORT_SYMBOL(skb_copy_and_csum_bits);
3495 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
3499 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
3500 /* See comments in __skb_checksum_complete(). */
3502 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3503 !skb->csum_complete_sw)
3504 netdev_rx_csum_fault(skb->dev, skb);
3506 if (!skb_shared(skb))
3507 skb->csum_valid = !sum;
3510 EXPORT_SYMBOL(__skb_checksum_complete_head);
3512 /* This function assumes skb->csum already holds pseudo header's checksum,
3513 * which has been changed from the hardware checksum, for example, by
3514 * __skb_checksum_validate_complete(). And, the original skb->csum must
3515 * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
3517 * It returns non-zero if the recomputed checksum is still invalid, otherwise
3518 * zero. The new checksum is stored back into skb->csum unless the skb is
3521 __sum16 __skb_checksum_complete(struct sk_buff *skb)
3526 csum = skb_checksum(skb, 0, skb->len, 0);
3528 sum = csum_fold(csum_add(skb->csum, csum));
3529 /* This check is inverted, because we already knew the hardware
3530 * checksum is invalid before calling this function. So, if the
3531 * re-computed checksum is valid instead, then we have a mismatch
3532 * between the original skb->csum and skb_checksum(). This means either
3533 * the original hardware checksum is incorrect or we screw up skb->csum
3534 * when moving skb->data around.
3537 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3538 !skb->csum_complete_sw)
3539 netdev_rx_csum_fault(skb->dev, skb);
3542 if (!skb_shared(skb)) {
3543 /* Save full packet checksum */
3545 skb->ip_summed = CHECKSUM_COMPLETE;
3546 skb->csum_complete_sw = 1;
3547 skb->csum_valid = !sum;
3552 EXPORT_SYMBOL(__skb_checksum_complete);
3554 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
3556 net_warn_ratelimited(
3557 "%s: attempt to compute crc32c without libcrc32c.ko\n",
3562 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
3563 int offset, int len)
3565 net_warn_ratelimited(
3566 "%s: attempt to compute crc32c without libcrc32c.ko\n",
3571 static const struct skb_checksum_ops default_crc32c_ops = {
3572 .update = warn_crc32c_csum_update,
3573 .combine = warn_crc32c_csum_combine,
3576 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
3577 &default_crc32c_ops;
3578 EXPORT_SYMBOL(crc32c_csum_stub);
3581 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
3582 * @from: source buffer
3584 * Calculates the amount of linear headroom needed in the 'to' skb passed
3585 * into skb_zerocopy().
3588 skb_zerocopy_headlen(const struct sk_buff *from)
3590 unsigned int hlen = 0;
3592 if (!from->head_frag ||
3593 skb_headlen(from) < L1_CACHE_BYTES ||
3594 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) {
3595 hlen = skb_headlen(from);
3600 if (skb_has_frag_list(from))
3605 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
3608 * skb_zerocopy - Zero copy skb to skb
3609 * @to: destination buffer
3610 * @from: source buffer
3611 * @len: number of bytes to copy from source buffer
3612 * @hlen: size of linear headroom in destination buffer
3614 * Copies up to `len` bytes from `from` to `to` by creating references
3615 * to the frags in the source buffer.
3617 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the
3618 * headroom in the `to` buffer.
3621 * 0: everything is OK
3622 * -ENOMEM: couldn't orphan frags of @from due to lack of memory
3623 * -EFAULT: skb_copy_bits() found some problem with skb geometry
3626 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
3629 int plen = 0; /* length of skb->head fragment */
3632 unsigned int offset;
3634 BUG_ON(!from->head_frag && !hlen);
3636 /* dont bother with small payloads */
3637 if (len <= skb_tailroom(to))
3638 return skb_copy_bits(from, 0, skb_put(to, len), len);
3641 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
3646 plen = min_t(int, skb_headlen(from), len);
3648 page = virt_to_head_page(from->head);
3649 offset = from->data - (unsigned char *)page_address(page);
3650 __skb_fill_page_desc(to, 0, page, offset, plen);
3657 skb_len_add(to, len + plen);
3659 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
3663 skb_zerocopy_clone(to, from, GFP_ATOMIC);
3665 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
3670 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
3671 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
3673 skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
3675 skb_frag_ref(to, j);
3678 skb_shinfo(to)->nr_frags = j;
3682 EXPORT_SYMBOL_GPL(skb_zerocopy);
3684 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
3689 if (skb->ip_summed == CHECKSUM_PARTIAL)
3690 csstart = skb_checksum_start_offset(skb);
3692 csstart = skb_headlen(skb);
3694 BUG_ON(csstart > skb_headlen(skb));
3696 skb_copy_from_linear_data(skb, to, csstart);
3699 if (csstart != skb->len)
3700 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
3701 skb->len - csstart);
3703 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3704 long csstuff = csstart + skb->csum_offset;
3706 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
3709 EXPORT_SYMBOL(skb_copy_and_csum_dev);
3712 * skb_dequeue - remove from the head of the queue
3713 * @list: list to dequeue from
3715 * Remove the head of the list. The list lock is taken so the function
3716 * may be used safely with other locking list functions. The head item is
3717 * returned or %NULL if the list is empty.
3720 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
3722 unsigned long flags;
3723 struct sk_buff *result;
3725 spin_lock_irqsave(&list->lock, flags);
3726 result = __skb_dequeue(list);
3727 spin_unlock_irqrestore(&list->lock, flags);
3730 EXPORT_SYMBOL(skb_dequeue);
3733 * skb_dequeue_tail - remove from the tail of the queue
3734 * @list: list to dequeue from
3736 * Remove the tail of the list. The list lock is taken so the function
3737 * may be used safely with other locking list functions. The tail item is
3738 * returned or %NULL if the list is empty.
3740 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
3742 unsigned long flags;
3743 struct sk_buff *result;
3745 spin_lock_irqsave(&list->lock, flags);
3746 result = __skb_dequeue_tail(list);
3747 spin_unlock_irqrestore(&list->lock, flags);
3750 EXPORT_SYMBOL(skb_dequeue_tail);
3753 * skb_queue_purge_reason - empty a list
3754 * @list: list to empty
3755 * @reason: drop reason
3757 * Delete all buffers on an &sk_buff list. Each buffer is removed from
3758 * the list and one reference dropped. This function takes the list
3759 * lock and is atomic with respect to other list locking functions.
3761 void skb_queue_purge_reason(struct sk_buff_head *list,
3762 enum skb_drop_reason reason)
3764 struct sk_buff_head tmp;
3765 unsigned long flags;
3767 if (skb_queue_empty_lockless(list))
3770 __skb_queue_head_init(&tmp);
3772 spin_lock_irqsave(&list->lock, flags);
3773 skb_queue_splice_init(list, &tmp);
3774 spin_unlock_irqrestore(&list->lock, flags);
3776 __skb_queue_purge_reason(&tmp, reason);
3778 EXPORT_SYMBOL(skb_queue_purge_reason);
3781 * skb_rbtree_purge - empty a skb rbtree
3782 * @root: root of the rbtree to empty
3783 * Return value: the sum of truesizes of all purged skbs.
3785 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
3786 * the list and one reference dropped. This function does not take
3787 * any lock. Synchronization should be handled by the caller (e.g., TCP
3788 * out-of-order queue is protected by the socket lock).
3790 unsigned int skb_rbtree_purge(struct rb_root *root)
3792 struct rb_node *p = rb_first(root);
3793 unsigned int sum = 0;
3796 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
3799 rb_erase(&skb->rbnode, root);
3800 sum += skb->truesize;
3806 void skb_errqueue_purge(struct sk_buff_head *list)
3808 struct sk_buff *skb, *next;
3809 struct sk_buff_head kill;
3810 unsigned long flags;
3812 __skb_queue_head_init(&kill);
3814 spin_lock_irqsave(&list->lock, flags);
3815 skb_queue_walk_safe(list, skb, next) {
3816 if (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY ||
3817 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING)
3819 __skb_unlink(skb, list);
3820 __skb_queue_tail(&kill, skb);
3822 spin_unlock_irqrestore(&list->lock, flags);
3823 __skb_queue_purge(&kill);
3825 EXPORT_SYMBOL(skb_errqueue_purge);
3828 * skb_queue_head - queue a buffer at the list head
3829 * @list: list to use
3830 * @newsk: buffer to queue
3832 * Queue a buffer at the start of the list. This function takes the
3833 * list lock and can be used safely with other locking &sk_buff functions
3836 * A buffer cannot be placed on two lists at the same time.
3838 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
3840 unsigned long flags;
3842 spin_lock_irqsave(&list->lock, flags);
3843 __skb_queue_head(list, newsk);
3844 spin_unlock_irqrestore(&list->lock, flags);
3846 EXPORT_SYMBOL(skb_queue_head);
3849 * skb_queue_tail - queue a buffer at the list tail
3850 * @list: list to use
3851 * @newsk: buffer to queue
3853 * Queue a buffer at the tail of the list. This function takes the
3854 * list lock and can be used safely with other locking &sk_buff functions
3857 * A buffer cannot be placed on two lists at the same time.
3859 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
3861 unsigned long flags;
3863 spin_lock_irqsave(&list->lock, flags);
3864 __skb_queue_tail(list, newsk);
3865 spin_unlock_irqrestore(&list->lock, flags);
3867 EXPORT_SYMBOL(skb_queue_tail);
3870 * skb_unlink - remove a buffer from a list
3871 * @skb: buffer to remove
3872 * @list: list to use
3874 * Remove a packet from a list. The list locks are taken and this
3875 * function is atomic with respect to other list locked calls
3877 * You must know what list the SKB is on.
3879 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
3881 unsigned long flags;
3883 spin_lock_irqsave(&list->lock, flags);
3884 __skb_unlink(skb, list);
3885 spin_unlock_irqrestore(&list->lock, flags);
3887 EXPORT_SYMBOL(skb_unlink);
3890 * skb_append - append a buffer
3891 * @old: buffer to insert after
3892 * @newsk: buffer to insert
3893 * @list: list to use
3895 * Place a packet after a given packet in a list. The list locks are taken
3896 * and this function is atomic with respect to other list locked calls.
3897 * A buffer cannot be placed on two lists at the same time.
3899 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
3901 unsigned long flags;
3903 spin_lock_irqsave(&list->lock, flags);
3904 __skb_queue_after(list, old, newsk);
3905 spin_unlock_irqrestore(&list->lock, flags);
3907 EXPORT_SYMBOL(skb_append);
3909 static inline void skb_split_inside_header(struct sk_buff *skb,
3910 struct sk_buff* skb1,
3911 const u32 len, const int pos)
3915 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
3917 /* And move data appendix as is. */
3918 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
3919 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
3921 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
3922 skb_shinfo(skb)->nr_frags = 0;
3923 skb1->data_len = skb->data_len;
3924 skb1->len += skb1->data_len;
3927 skb_set_tail_pointer(skb, len);
3930 static inline void skb_split_no_header(struct sk_buff *skb,
3931 struct sk_buff* skb1,
3932 const u32 len, int pos)
3935 const int nfrags = skb_shinfo(skb)->nr_frags;
3937 skb_shinfo(skb)->nr_frags = 0;
3938 skb1->len = skb1->data_len = skb->len - len;
3940 skb->data_len = len - pos;
3942 for (i = 0; i < nfrags; i++) {
3943 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
3945 if (pos + size > len) {
3946 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3950 * We have two variants in this case:
3951 * 1. Move all the frag to the second
3952 * part, if it is possible. F.e.
3953 * this approach is mandatory for TUX,
3954 * where splitting is expensive.
3955 * 2. Split is accurately. We make this.
3957 skb_frag_ref(skb, i);
3958 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
3959 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3960 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3961 skb_shinfo(skb)->nr_frags++;
3965 skb_shinfo(skb)->nr_frags++;
3968 skb_shinfo(skb1)->nr_frags = k;
3972 * skb_split - Split fragmented skb to two parts at length len.
3973 * @skb: the buffer to split
3974 * @skb1: the buffer to receive the second part
3975 * @len: new length for skb
3977 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3979 int pos = skb_headlen(skb);
3980 const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY;
3982 skb_zcopy_downgrade_managed(skb);
3984 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags;
3985 skb_zerocopy_clone(skb1, skb, 0);
3986 if (len < pos) /* Split line is inside header. */
3987 skb_split_inside_header(skb, skb1, len, pos);
3988 else /* Second chunk has no header, nothing to copy. */
3989 skb_split_no_header(skb, skb1, len, pos);
3991 EXPORT_SYMBOL(skb_split);
3993 /* Shifting from/to a cloned skb is a no-go.
3995 * Caller cannot keep skb_shinfo related pointers past calling here!
3997 static int skb_prepare_for_shift(struct sk_buff *skb)
3999 return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
4003 * skb_shift - Shifts paged data partially from skb to another
4004 * @tgt: buffer into which tail data gets added
4005 * @skb: buffer from which the paged data comes from
4006 * @shiftlen: shift up to this many bytes
4008 * Attempts to shift up to shiftlen worth of bytes, which may be less than
4009 * the length of the skb, from skb to tgt. Returns number bytes shifted.
4010 * It's up to caller to free skb if everything was shifted.
4012 * If @tgt runs out of frags, the whole operation is aborted.
4014 * Skb cannot include anything else but paged data while tgt is allowed
4015 * to have non-paged data as well.
4017 * TODO: full sized shift could be optimized but that would need
4018 * specialized skb free'er to handle frags without up-to-date nr_frags.
4020 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
4022 int from, to, merge, todo;
4023 skb_frag_t *fragfrom, *fragto;
4025 BUG_ON(shiftlen > skb->len);
4027 if (skb_headlen(skb))
4029 if (skb_zcopy(tgt) || skb_zcopy(skb))
4034 to = skb_shinfo(tgt)->nr_frags;
4035 fragfrom = &skb_shinfo(skb)->frags[from];
4037 /* Actual merge is delayed until the point when we know we can
4038 * commit all, so that we don't have to undo partial changes
4041 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
4042 skb_frag_off(fragfrom))) {
4047 todo -= skb_frag_size(fragfrom);
4049 if (skb_prepare_for_shift(skb) ||
4050 skb_prepare_for_shift(tgt))
4053 /* All previous frag pointers might be stale! */
4054 fragfrom = &skb_shinfo(skb)->frags[from];
4055 fragto = &skb_shinfo(tgt)->frags[merge];
4057 skb_frag_size_add(fragto, shiftlen);
4058 skb_frag_size_sub(fragfrom, shiftlen);
4059 skb_frag_off_add(fragfrom, shiftlen);
4067 /* Skip full, not-fitting skb to avoid expensive operations */
4068 if ((shiftlen == skb->len) &&
4069 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
4072 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
4075 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
4076 if (to == MAX_SKB_FRAGS)
4079 fragfrom = &skb_shinfo(skb)->frags[from];
4080 fragto = &skb_shinfo(tgt)->frags[to];
4082 if (todo >= skb_frag_size(fragfrom)) {
4083 *fragto = *fragfrom;
4084 todo -= skb_frag_size(fragfrom);
4089 __skb_frag_ref(fragfrom);
4090 skb_frag_page_copy(fragto, fragfrom);
4091 skb_frag_off_copy(fragto, fragfrom);
4092 skb_frag_size_set(fragto, todo);
4094 skb_frag_off_add(fragfrom, todo);
4095 skb_frag_size_sub(fragfrom, todo);
4103 /* Ready to "commit" this state change to tgt */
4104 skb_shinfo(tgt)->nr_frags = to;
4107 fragfrom = &skb_shinfo(skb)->frags[0];
4108 fragto = &skb_shinfo(tgt)->frags[merge];
4110 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
4111 __skb_frag_unref(fragfrom, skb->pp_recycle);
4114 /* Reposition in the original skb */
4116 while (from < skb_shinfo(skb)->nr_frags)
4117 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
4118 skb_shinfo(skb)->nr_frags = to;
4120 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
4123 /* Most likely the tgt won't ever need its checksum anymore, skb on
4124 * the other hand might need it if it needs to be resent
4126 tgt->ip_summed = CHECKSUM_PARTIAL;
4127 skb->ip_summed = CHECKSUM_PARTIAL;
4129 skb_len_add(skb, -shiftlen);
4130 skb_len_add(tgt, shiftlen);
4136 * skb_prepare_seq_read - Prepare a sequential read of skb data
4137 * @skb: the buffer to read
4138 * @from: lower offset of data to be read
4139 * @to: upper offset of data to be read
4140 * @st: state variable
4142 * Initializes the specified state variable. Must be called before
4143 * invoking skb_seq_read() for the first time.
4145 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
4146 unsigned int to, struct skb_seq_state *st)
4148 st->lower_offset = from;
4149 st->upper_offset = to;
4150 st->root_skb = st->cur_skb = skb;
4151 st->frag_idx = st->stepped_offset = 0;
4152 st->frag_data = NULL;
4155 EXPORT_SYMBOL(skb_prepare_seq_read);
4158 * skb_seq_read - Sequentially read skb data
4159 * @consumed: number of bytes consumed by the caller so far
4160 * @data: destination pointer for data to be returned
4161 * @st: state variable
4163 * Reads a block of skb data at @consumed relative to the
4164 * lower offset specified to skb_prepare_seq_read(). Assigns
4165 * the head of the data block to @data and returns the length
4166 * of the block or 0 if the end of the skb data or the upper
4167 * offset has been reached.
4169 * The caller is not required to consume all of the data
4170 * returned, i.e. @consumed is typically set to the number
4171 * of bytes already consumed and the next call to
4172 * skb_seq_read() will return the remaining part of the block.
4174 * Note 1: The size of each block of data returned can be arbitrary,
4175 * this limitation is the cost for zerocopy sequential
4176 * reads of potentially non linear data.
4178 * Note 2: Fragment lists within fragments are not implemented
4179 * at the moment, state->root_skb could be replaced with
4180 * a stack for this purpose.
4182 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
4183 struct skb_seq_state *st)
4185 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
4188 if (unlikely(abs_offset >= st->upper_offset)) {
4189 if (st->frag_data) {
4190 kunmap_atomic(st->frag_data);
4191 st->frag_data = NULL;
4197 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
4199 if (abs_offset < block_limit && !st->frag_data) {
4200 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
4201 return block_limit - abs_offset;
4204 if (st->frag_idx == 0 && !st->frag_data)
4205 st->stepped_offset += skb_headlen(st->cur_skb);
4207 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
4208 unsigned int pg_idx, pg_off, pg_sz;
4210 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
4213 pg_off = skb_frag_off(frag);
4214 pg_sz = skb_frag_size(frag);
4216 if (skb_frag_must_loop(skb_frag_page(frag))) {
4217 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT;
4218 pg_off = offset_in_page(pg_off + st->frag_off);
4219 pg_sz = min_t(unsigned int, pg_sz - st->frag_off,
4220 PAGE_SIZE - pg_off);
4223 block_limit = pg_sz + st->stepped_offset;
4224 if (abs_offset < block_limit) {
4226 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx);
4228 *data = (u8 *)st->frag_data + pg_off +
4229 (abs_offset - st->stepped_offset);
4231 return block_limit - abs_offset;
4234 if (st->frag_data) {
4235 kunmap_atomic(st->frag_data);
4236 st->frag_data = NULL;
4239 st->stepped_offset += pg_sz;
4240 st->frag_off += pg_sz;
4241 if (st->frag_off == skb_frag_size(frag)) {
4247 if (st->frag_data) {
4248 kunmap_atomic(st->frag_data);
4249 st->frag_data = NULL;
4252 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
4253 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
4256 } else if (st->cur_skb->next) {
4257 st->cur_skb = st->cur_skb->next;
4264 EXPORT_SYMBOL(skb_seq_read);
4267 * skb_abort_seq_read - Abort a sequential read of skb data
4268 * @st: state variable
4270 * Must be called if skb_seq_read() was not called until it
4273 void skb_abort_seq_read(struct skb_seq_state *st)
4276 kunmap_atomic(st->frag_data);
4278 EXPORT_SYMBOL(skb_abort_seq_read);
4280 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
4282 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
4283 struct ts_config *conf,
4284 struct ts_state *state)
4286 return skb_seq_read(offset, text, TS_SKB_CB(state));
4289 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
4291 skb_abort_seq_read(TS_SKB_CB(state));
4295 * skb_find_text - Find a text pattern in skb data
4296 * @skb: the buffer to look in
4297 * @from: search offset
4299 * @config: textsearch configuration
4301 * Finds a pattern in the skb data according to the specified
4302 * textsearch configuration. Use textsearch_next() to retrieve
4303 * subsequent occurrences of the pattern. Returns the offset
4304 * to the first occurrence or UINT_MAX if no match was found.
4306 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
4307 unsigned int to, struct ts_config *config)
4309 unsigned int patlen = config->ops->get_pattern_len(config);
4310 struct ts_state state;
4313 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb));
4315 config->get_next_block = skb_ts_get_next_block;
4316 config->finish = skb_ts_finish;
4318 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
4320 ret = textsearch_find(config, &state);
4321 return (ret + patlen <= to - from ? ret : UINT_MAX);
4323 EXPORT_SYMBOL(skb_find_text);
4325 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
4326 int offset, size_t size, size_t max_frags)
4328 int i = skb_shinfo(skb)->nr_frags;
4330 if (skb_can_coalesce(skb, i, page, offset)) {
4331 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
4332 } else if (i < max_frags) {
4333 skb_zcopy_downgrade_managed(skb);
4335 skb_fill_page_desc_noacc(skb, i, page, offset, size);
4342 EXPORT_SYMBOL_GPL(skb_append_pagefrags);
4345 * skb_pull_rcsum - pull skb and update receive checksum
4346 * @skb: buffer to update
4347 * @len: length of data pulled
4349 * This function performs an skb_pull on the packet and updates
4350 * the CHECKSUM_COMPLETE checksum. It should be used on
4351 * receive path processing instead of skb_pull unless you know
4352 * that the checksum difference is zero (e.g., a valid IP header)
4353 * or you are setting ip_summed to CHECKSUM_NONE.
4355 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
4357 unsigned char *data = skb->data;
4359 BUG_ON(len > skb->len);
4360 __skb_pull(skb, len);
4361 skb_postpull_rcsum(skb, data, len);
4364 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
4366 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
4368 skb_frag_t head_frag;
4371 page = virt_to_head_page(frag_skb->head);
4372 skb_frag_fill_page_desc(&head_frag, page, frag_skb->data -
4373 (unsigned char *)page_address(page),
4374 skb_headlen(frag_skb));
4378 struct sk_buff *skb_segment_list(struct sk_buff *skb,
4379 netdev_features_t features,
4380 unsigned int offset)
4382 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
4383 unsigned int tnl_hlen = skb_tnl_header_len(skb);
4384 unsigned int delta_truesize = 0;
4385 unsigned int delta_len = 0;
4386 struct sk_buff *tail = NULL;
4387 struct sk_buff *nskb, *tmp;
4390 skb_push(skb, -skb_network_offset(skb) + offset);
4392 /* Ensure the head is writeable before touching the shared info */
4393 err = skb_unclone(skb, GFP_ATOMIC);
4397 skb_shinfo(skb)->frag_list = NULL;
4401 list_skb = list_skb->next;
4404 delta_truesize += nskb->truesize;
4405 if (skb_shared(nskb)) {
4406 tmp = skb_clone(nskb, GFP_ATOMIC);
4410 err = skb_unclone(nskb, GFP_ATOMIC);
4421 if (unlikely(err)) {
4422 nskb->next = list_skb;
4428 delta_len += nskb->len;
4430 skb_push(nskb, -skb_network_offset(nskb) + offset);
4432 skb_release_head_state(nskb);
4433 len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb);
4434 __copy_skb_header(nskb, skb);
4436 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
4437 nskb->transport_header += len_diff;
4438 skb_copy_from_linear_data_offset(skb, -tnl_hlen,
4439 nskb->data - tnl_hlen,
4442 if (skb_needs_linearize(nskb, features) &&
4443 __skb_linearize(nskb))
4447 skb->truesize = skb->truesize - delta_truesize;
4448 skb->data_len = skb->data_len - delta_len;
4449 skb->len = skb->len - delta_len;
4455 if (skb_needs_linearize(skb, features) &&
4456 __skb_linearize(skb))
4464 kfree_skb_list(skb->next);
4466 return ERR_PTR(-ENOMEM);
4468 EXPORT_SYMBOL_GPL(skb_segment_list);
4471 * skb_segment - Perform protocol segmentation on skb.
4472 * @head_skb: buffer to segment
4473 * @features: features for the output path (see dev->features)
4475 * This function performs segmentation on the given skb. It returns
4476 * a pointer to the first in a list of new skbs for the segments.
4477 * In case of error it returns ERR_PTR(err).
4479 struct sk_buff *skb_segment(struct sk_buff *head_skb,
4480 netdev_features_t features)
4482 struct sk_buff *segs = NULL;
4483 struct sk_buff *tail = NULL;
4484 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
4485 unsigned int mss = skb_shinfo(head_skb)->gso_size;
4486 unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
4487 unsigned int offset = doffset;
4488 unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
4489 unsigned int partial_segs = 0;
4490 unsigned int headroom;
4491 unsigned int len = head_skb->len;
4492 struct sk_buff *frag_skb;
4500 if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) &&
4501 mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) {
4502 struct sk_buff *check_skb;
4504 for (check_skb = list_skb; check_skb; check_skb = check_skb->next) {
4505 if (skb_headlen(check_skb) && !check_skb->head_frag) {
4506 /* gso_size is untrusted, and we have a frag_list with
4507 * a linear non head_frag item.
4509 * If head_skb's headlen does not fit requested gso_size,
4510 * it means that the frag_list members do NOT terminate
4511 * on exact gso_size boundaries. Hence we cannot perform
4512 * skb_frag_t page sharing. Therefore we must fallback to
4513 * copying the frag_list skbs; we do so by disabling SG.
4515 features &= ~NETIF_F_SG;
4521 __skb_push(head_skb, doffset);
4522 proto = skb_network_protocol(head_skb, NULL);
4523 if (unlikely(!proto))
4524 return ERR_PTR(-EINVAL);
4526 sg = !!(features & NETIF_F_SG);
4527 csum = !!can_checksum_protocol(features, proto);
4529 if (sg && csum && (mss != GSO_BY_FRAGS)) {
4530 if (!(features & NETIF_F_GSO_PARTIAL)) {
4531 struct sk_buff *iter;
4532 unsigned int frag_len;
4535 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
4538 /* If we get here then all the required
4539 * GSO features except frag_list are supported.
4540 * Try to split the SKB to multiple GSO SKBs
4541 * with no frag_list.
4542 * Currently we can do that only when the buffers don't
4543 * have a linear part and all the buffers except
4544 * the last are of the same length.
4546 frag_len = list_skb->len;
4547 skb_walk_frags(head_skb, iter) {
4548 if (frag_len != iter->len && iter->next)
4550 if (skb_headlen(iter) && !iter->head_frag)
4556 if (len != frag_len)
4560 /* GSO partial only requires that we trim off any excess that
4561 * doesn't fit into an MSS sized block, so take care of that
4563 * Cap len to not accidentally hit GSO_BY_FRAGS.
4565 partial_segs = min(len, GSO_BY_FRAGS - 1) / mss;
4566 if (partial_segs > 1)
4567 mss *= partial_segs;
4573 headroom = skb_headroom(head_skb);
4574 pos = skb_headlen(head_skb);
4576 if (skb_orphan_frags(head_skb, GFP_ATOMIC))
4577 return ERR_PTR(-ENOMEM);
4579 nfrags = skb_shinfo(head_skb)->nr_frags;
4580 frag = skb_shinfo(head_skb)->frags;
4581 frag_skb = head_skb;
4584 struct sk_buff *nskb;
4585 skb_frag_t *nskb_frag;
4589 if (unlikely(mss == GSO_BY_FRAGS)) {
4590 len = list_skb->len;
4592 len = head_skb->len - offset;
4597 hsize = skb_headlen(head_skb) - offset;
4599 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) &&
4600 (skb_headlen(list_skb) == len || sg)) {
4601 BUG_ON(skb_headlen(list_skb) > len);
4603 nskb = skb_clone(list_skb, GFP_ATOMIC);
4604 if (unlikely(!nskb))
4608 nfrags = skb_shinfo(list_skb)->nr_frags;
4609 frag = skb_shinfo(list_skb)->frags;
4610 frag_skb = list_skb;
4611 pos += skb_headlen(list_skb);
4613 while (pos < offset + len) {
4614 BUG_ON(i >= nfrags);
4616 size = skb_frag_size(frag);
4617 if (pos + size > offset + len)
4625 list_skb = list_skb->next;
4627 if (unlikely(pskb_trim(nskb, len))) {
4632 hsize = skb_end_offset(nskb);
4633 if (skb_cow_head(nskb, doffset + headroom)) {
4638 nskb->truesize += skb_end_offset(nskb) - hsize;
4639 skb_release_head_state(nskb);
4640 __skb_push(nskb, doffset);
4644 if (hsize > len || !sg)
4647 nskb = __alloc_skb(hsize + doffset + headroom,
4648 GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
4651 if (unlikely(!nskb))
4654 skb_reserve(nskb, headroom);
4655 __skb_put(nskb, doffset);
4664 __copy_skb_header(nskb, head_skb);
4666 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
4667 skb_reset_mac_len(nskb);
4669 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
4670 nskb->data - tnl_hlen,
4671 doffset + tnl_hlen);
4673 if (nskb->len == len + doffset)
4674 goto perform_csum_check;
4678 if (!nskb->remcsum_offload)
4679 nskb->ip_summed = CHECKSUM_NONE;
4680 SKB_GSO_CB(nskb)->csum =
4681 skb_copy_and_csum_bits(head_skb, offset,
4685 SKB_GSO_CB(nskb)->csum_start =
4686 skb_headroom(nskb) + doffset;
4688 if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len))
4694 nskb_frag = skb_shinfo(nskb)->frags;
4696 skb_copy_from_linear_data_offset(head_skb, offset,
4697 skb_put(nskb, hsize), hsize);
4699 skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags &
4702 if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
4705 while (pos < offset + len) {
4707 if (skb_orphan_frags(list_skb, GFP_ATOMIC) ||
4708 skb_zerocopy_clone(nskb, list_skb,
4713 nfrags = skb_shinfo(list_skb)->nr_frags;
4714 frag = skb_shinfo(list_skb)->frags;
4715 frag_skb = list_skb;
4716 if (!skb_headlen(list_skb)) {
4719 BUG_ON(!list_skb->head_frag);
4721 /* to make room for head_frag. */
4726 list_skb = list_skb->next;
4729 if (unlikely(skb_shinfo(nskb)->nr_frags >=
4731 net_warn_ratelimited(
4732 "skb_segment: too many frags: %u %u\n",
4738 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
4739 __skb_frag_ref(nskb_frag);
4740 size = skb_frag_size(nskb_frag);
4743 skb_frag_off_add(nskb_frag, offset - pos);
4744 skb_frag_size_sub(nskb_frag, offset - pos);
4747 skb_shinfo(nskb)->nr_frags++;
4749 if (pos + size <= offset + len) {
4754 skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
4762 nskb->data_len = len - hsize;
4763 nskb->len += nskb->data_len;
4764 nskb->truesize += nskb->data_len;
4768 if (skb_has_shared_frag(nskb) &&
4769 __skb_linearize(nskb))
4772 if (!nskb->remcsum_offload)
4773 nskb->ip_summed = CHECKSUM_NONE;
4774 SKB_GSO_CB(nskb)->csum =
4775 skb_checksum(nskb, doffset,
4776 nskb->len - doffset, 0);
4777 SKB_GSO_CB(nskb)->csum_start =
4778 skb_headroom(nskb) + doffset;
4780 } while ((offset += len) < head_skb->len);
4782 /* Some callers want to get the end of the list.
4783 * Put it in segs->prev to avoid walking the list.
4784 * (see validate_xmit_skb_list() for example)
4789 struct sk_buff *iter;
4790 int type = skb_shinfo(head_skb)->gso_type;
4791 unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
4793 /* Update type to add partial and then remove dodgy if set */
4794 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
4795 type &= ~SKB_GSO_DODGY;
4797 /* Update GSO info and prepare to start updating headers on
4798 * our way back down the stack of protocols.
4800 for (iter = segs; iter; iter = iter->next) {
4801 skb_shinfo(iter)->gso_size = gso_size;
4802 skb_shinfo(iter)->gso_segs = partial_segs;
4803 skb_shinfo(iter)->gso_type = type;
4804 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
4807 if (tail->len - doffset <= gso_size)
4808 skb_shinfo(tail)->gso_size = 0;
4809 else if (tail != segs)
4810 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
4813 /* Following permits correct backpressure, for protocols
4814 * using skb_set_owner_w().
4815 * Idea is to tranfert ownership from head_skb to last segment.
4817 if (head_skb->destructor == sock_wfree) {
4818 swap(tail->truesize, head_skb->truesize);
4819 swap(tail->destructor, head_skb->destructor);
4820 swap(tail->sk, head_skb->sk);
4825 kfree_skb_list(segs);
4826 return ERR_PTR(err);
4828 EXPORT_SYMBOL_GPL(skb_segment);
4830 #ifdef CONFIG_SKB_EXTENSIONS
4831 #define SKB_EXT_ALIGN_VALUE 8
4832 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
4834 static const u8 skb_ext_type_len[] = {
4835 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4836 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
4839 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
4841 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
4842 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
4844 #if IS_ENABLED(CONFIG_MPTCP)
4845 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
4847 #if IS_ENABLED(CONFIG_MCTP_FLOWS)
4848 [SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow),
4852 static __always_inline unsigned int skb_ext_total_length(void)
4854 unsigned int l = SKB_EXT_CHUNKSIZEOF(struct skb_ext);
4857 for (i = 0; i < ARRAY_SIZE(skb_ext_type_len); i++)
4858 l += skb_ext_type_len[i];
4863 static void skb_extensions_init(void)
4865 BUILD_BUG_ON(SKB_EXT_NUM >= 8);
4866 #if !IS_ENABLED(CONFIG_KCOV_INSTRUMENT_ALL)
4867 BUILD_BUG_ON(skb_ext_total_length() > 255);
4870 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
4871 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
4873 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4877 static void skb_extensions_init(void) {}
4880 /* The SKB kmem_cache slab is critical for network performance. Never
4881 * merge/alias the slab with similar sized objects. This avoids fragmentation
4882 * that hurts performance of kmem_cache_{alloc,free}_bulk APIs.
4884 #ifndef CONFIG_SLUB_TINY
4885 #define FLAG_SKB_NO_MERGE SLAB_NO_MERGE
4886 #else /* CONFIG_SLUB_TINY - simple loop in kmem_cache_alloc_bulk */
4887 #define FLAG_SKB_NO_MERGE 0
4890 void __init skb_init(void)
4892 skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
4893 sizeof(struct sk_buff),
4895 SLAB_HWCACHE_ALIGN|SLAB_PANIC|
4897 offsetof(struct sk_buff, cb),
4898 sizeof_field(struct sk_buff, cb),
4900 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
4901 sizeof(struct sk_buff_fclones),
4903 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4905 /* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes.
4906 * struct skb_shared_info is located at the end of skb->head,
4907 * and should not be copied to/from user.
4909 skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
4910 SKB_SMALL_HEAD_CACHE_SIZE,
4912 SLAB_HWCACHE_ALIGN | SLAB_PANIC,
4914 SKB_SMALL_HEAD_HEADROOM,
4916 skb_extensions_init();
4920 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
4921 unsigned int recursion_level)
4923 int start = skb_headlen(skb);
4924 int i, copy = start - offset;
4925 struct sk_buff *frag_iter;
4928 if (unlikely(recursion_level >= 24))
4934 sg_set_buf(sg, skb->data + offset, copy);
4936 if ((len -= copy) == 0)
4941 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4944 WARN_ON(start > offset + len);
4946 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
4947 if ((copy = end - offset) > 0) {
4948 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4949 if (unlikely(elt && sg_is_last(&sg[elt - 1])))
4954 sg_set_page(&sg[elt], skb_frag_page(frag), copy,
4955 skb_frag_off(frag) + offset - start);
4964 skb_walk_frags(skb, frag_iter) {
4967 WARN_ON(start > offset + len);
4969 end = start + frag_iter->len;
4970 if ((copy = end - offset) > 0) {
4971 if (unlikely(elt && sg_is_last(&sg[elt - 1])))
4976 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
4977 copy, recursion_level + 1);
4978 if (unlikely(ret < 0))
4981 if ((len -= copy) == 0)
4992 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
4993 * @skb: Socket buffer containing the buffers to be mapped
4994 * @sg: The scatter-gather list to map into
4995 * @offset: The offset into the buffer's contents to start mapping
4996 * @len: Length of buffer space to be mapped
4998 * Fill the specified scatter-gather list with mappings/pointers into a
4999 * region of the buffer space attached to a socket buffer. Returns either
5000 * the number of scatterlist items used, or -EMSGSIZE if the contents
5003 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
5005 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
5010 sg_mark_end(&sg[nsg - 1]);
5014 EXPORT_SYMBOL_GPL(skb_to_sgvec);
5016 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
5017 * sglist without mark the sg which contain last skb data as the end.
5018 * So the caller can mannipulate sg list as will when padding new data after
5019 * the first call without calling sg_unmark_end to expend sg list.
5021 * Scenario to use skb_to_sgvec_nomark:
5023 * 2. skb_to_sgvec_nomark(payload1)
5024 * 3. skb_to_sgvec_nomark(payload2)
5026 * This is equivalent to:
5028 * 2. skb_to_sgvec(payload1)
5030 * 4. skb_to_sgvec(payload2)
5032 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
5033 * is more preferable.
5035 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
5036 int offset, int len)
5038 return __skb_to_sgvec(skb, sg, offset, len, 0);
5040 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
5045 * skb_cow_data - Check that a socket buffer's data buffers are writable
5046 * @skb: The socket buffer to check.
5047 * @tailbits: Amount of trailing space to be added
5048 * @trailer: Returned pointer to the skb where the @tailbits space begins
5050 * Make sure that the data buffers attached to a socket buffer are
5051 * writable. If they are not, private copies are made of the data buffers
5052 * and the socket buffer is set to use these instead.
5054 * If @tailbits is given, make sure that there is space to write @tailbits
5055 * bytes of data beyond current end of socket buffer. @trailer will be
5056 * set to point to the skb in which this space begins.
5058 * The number of scatterlist elements required to completely map the
5059 * COW'd and extended socket buffer will be returned.
5061 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
5065 struct sk_buff *skb1, **skb_p;
5067 /* If skb is cloned or its head is paged, reallocate
5068 * head pulling out all the pages (pages are considered not writable
5069 * at the moment even if they are anonymous).
5071 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
5072 !__pskb_pull_tail(skb, __skb_pagelen(skb)))
5075 /* Easy case. Most of packets will go this way. */
5076 if (!skb_has_frag_list(skb)) {
5077 /* A little of trouble, not enough of space for trailer.
5078 * This should not happen, when stack is tuned to generate
5079 * good frames. OK, on miss we reallocate and reserve even more
5080 * space, 128 bytes is fair. */
5082 if (skb_tailroom(skb) < tailbits &&
5083 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
5091 /* Misery. We are in troubles, going to mincer fragments... */
5094 skb_p = &skb_shinfo(skb)->frag_list;
5097 while ((skb1 = *skb_p) != NULL) {
5100 /* The fragment is partially pulled by someone,
5101 * this can happen on input. Copy it and everything
5104 if (skb_shared(skb1))
5107 /* If the skb is the last, worry about trailer. */
5109 if (skb1->next == NULL && tailbits) {
5110 if (skb_shinfo(skb1)->nr_frags ||
5111 skb_has_frag_list(skb1) ||
5112 skb_tailroom(skb1) < tailbits)
5113 ntail = tailbits + 128;
5119 skb_shinfo(skb1)->nr_frags ||
5120 skb_has_frag_list(skb1)) {
5121 struct sk_buff *skb2;
5123 /* Fuck, we are miserable poor guys... */
5125 skb2 = skb_copy(skb1, GFP_ATOMIC);
5127 skb2 = skb_copy_expand(skb1,
5131 if (unlikely(skb2 == NULL))
5135 skb_set_owner_w(skb2, skb1->sk);
5137 /* Looking around. Are we still alive?
5138 * OK, link new skb, drop old one */
5140 skb2->next = skb1->next;
5147 skb_p = &skb1->next;
5152 EXPORT_SYMBOL_GPL(skb_cow_data);
5154 static void sock_rmem_free(struct sk_buff *skb)
5156 struct sock *sk = skb->sk;
5158 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
5161 static void skb_set_err_queue(struct sk_buff *skb)
5163 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
5164 * So, it is safe to (mis)use it to mark skbs on the error queue.
5166 skb->pkt_type = PACKET_OUTGOING;
5167 BUILD_BUG_ON(PACKET_OUTGOING == 0);
5171 * Note: We dont mem charge error packets (no sk_forward_alloc changes)
5173 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
5175 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
5176 (unsigned int)READ_ONCE(sk->sk_rcvbuf))
5181 skb->destructor = sock_rmem_free;
5182 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
5183 skb_set_err_queue(skb);
5185 /* before exiting rcu section, make sure dst is refcounted */
5188 skb_queue_tail(&sk->sk_error_queue, skb);
5189 if (!sock_flag(sk, SOCK_DEAD))
5190 sk_error_report(sk);
5193 EXPORT_SYMBOL(sock_queue_err_skb);
5195 static bool is_icmp_err_skb(const struct sk_buff *skb)
5197 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
5198 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
5201 struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
5203 struct sk_buff_head *q = &sk->sk_error_queue;
5204 struct sk_buff *skb, *skb_next = NULL;
5205 bool icmp_next = false;
5206 unsigned long flags;
5208 if (skb_queue_empty_lockless(q))
5211 spin_lock_irqsave(&q->lock, flags);
5212 skb = __skb_dequeue(q);
5213 if (skb && (skb_next = skb_peek(q))) {
5214 icmp_next = is_icmp_err_skb(skb_next);
5216 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno;
5218 spin_unlock_irqrestore(&q->lock, flags);
5220 if (is_icmp_err_skb(skb) && !icmp_next)
5224 sk_error_report(sk);
5228 EXPORT_SYMBOL(sock_dequeue_err_skb);
5231 * skb_clone_sk - create clone of skb, and take reference to socket
5232 * @skb: the skb to clone
5234 * This function creates a clone of a buffer that holds a reference on
5235 * sk_refcnt. Buffers created via this function are meant to be
5236 * returned using sock_queue_err_skb, or free via kfree_skb.
5238 * When passing buffers allocated with this function to sock_queue_err_skb
5239 * it is necessary to wrap the call with sock_hold/sock_put in order to
5240 * prevent the socket from being released prior to being enqueued on
5241 * the sk_error_queue.
5243 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
5245 struct sock *sk = skb->sk;
5246 struct sk_buff *clone;
5248 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
5251 clone = skb_clone(skb, GFP_ATOMIC);
5258 clone->destructor = sock_efree;
5262 EXPORT_SYMBOL(skb_clone_sk);
5264 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
5269 struct sock_exterr_skb *serr;
5272 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
5274 serr = SKB_EXT_ERR(skb);
5275 memset(serr, 0, sizeof(*serr));
5276 serr->ee.ee_errno = ENOMSG;
5277 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
5278 serr->ee.ee_info = tstype;
5279 serr->opt_stats = opt_stats;
5280 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
5281 if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) {
5282 serr->ee.ee_data = skb_shinfo(skb)->tskey;
5284 serr->ee.ee_data -= atomic_read(&sk->sk_tskey);
5287 err = sock_queue_err_skb(sk, skb);
5293 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
5297 if (likely(READ_ONCE(sysctl_tstamp_allow_data) || tsonly))
5300 read_lock_bh(&sk->sk_callback_lock);
5301 ret = sk->sk_socket && sk->sk_socket->file &&
5302 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
5303 read_unlock_bh(&sk->sk_callback_lock);
5307 void skb_complete_tx_timestamp(struct sk_buff *skb,
5308 struct skb_shared_hwtstamps *hwtstamps)
5310 struct sock *sk = skb->sk;
5312 if (!skb_may_tx_timestamp(sk, false))
5315 /* Take a reference to prevent skb_orphan() from freeing the socket,
5316 * but only if the socket refcount is not zero.
5318 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
5319 *skb_hwtstamps(skb) = *hwtstamps;
5320 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
5328 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
5330 void __skb_tstamp_tx(struct sk_buff *orig_skb,
5331 const struct sk_buff *ack_skb,
5332 struct skb_shared_hwtstamps *hwtstamps,
5333 struct sock *sk, int tstype)
5335 struct sk_buff *skb;
5336 bool tsonly, opt_stats = false;
5342 tsflags = READ_ONCE(sk->sk_tsflags);
5343 if (!hwtstamps && !(tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
5344 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
5347 tsonly = tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
5348 if (!skb_may_tx_timestamp(sk, tsonly))
5353 if ((tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
5355 skb = tcp_get_timestamping_opt_stats(sk, orig_skb,
5360 skb = alloc_skb(0, GFP_ATOMIC);
5362 skb = skb_clone(orig_skb, GFP_ATOMIC);
5364 if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) {
5373 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
5375 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
5379 *skb_hwtstamps(skb) = *hwtstamps;
5381 __net_timestamp(skb);
5383 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
5385 EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
5387 void skb_tstamp_tx(struct sk_buff *orig_skb,
5388 struct skb_shared_hwtstamps *hwtstamps)
5390 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk,
5393 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
5395 #ifdef CONFIG_WIRELESS
5396 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
5398 struct sock *sk = skb->sk;
5399 struct sock_exterr_skb *serr;
5402 skb->wifi_acked_valid = 1;
5403 skb->wifi_acked = acked;
5405 serr = SKB_EXT_ERR(skb);
5406 memset(serr, 0, sizeof(*serr));
5407 serr->ee.ee_errno = ENOMSG;
5408 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
5410 /* Take a reference to prevent skb_orphan() from freeing the socket,
5411 * but only if the socket refcount is not zero.
5413 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
5414 err = sock_queue_err_skb(sk, skb);
5420 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
5421 #endif /* CONFIG_WIRELESS */
5424 * skb_partial_csum_set - set up and verify partial csum values for packet
5425 * @skb: the skb to set
5426 * @start: the number of bytes after skb->data to start checksumming.
5427 * @off: the offset from start to place the checksum.
5429 * For untrusted partially-checksummed packets, we need to make sure the values
5430 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
5432 * This function checks and sets those values and skb->ip_summed: if this
5433 * returns false you should drop the packet.
5435 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
5437 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
5438 u32 csum_start = skb_headroom(skb) + (u32)start;
5440 if (unlikely(csum_start >= U16_MAX || csum_end > skb_headlen(skb))) {
5441 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
5442 start, off, skb_headroom(skb), skb_headlen(skb));
5445 skb->ip_summed = CHECKSUM_PARTIAL;
5446 skb->csum_start = csum_start;
5447 skb->csum_offset = off;
5448 skb->transport_header = csum_start;
5451 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
5453 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
5456 if (skb_headlen(skb) >= len)
5459 /* If we need to pullup then pullup to the max, so we
5460 * won't need to do it again.
5465 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
5468 if (skb_headlen(skb) < len)
5474 #define MAX_TCP_HDR_LEN (15 * 4)
5476 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
5477 typeof(IPPROTO_IP) proto,
5484 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
5485 off + MAX_TCP_HDR_LEN);
5486 if (!err && !skb_partial_csum_set(skb, off,
5487 offsetof(struct tcphdr,
5490 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
5493 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
5494 off + sizeof(struct udphdr));
5495 if (!err && !skb_partial_csum_set(skb, off,
5496 offsetof(struct udphdr,
5499 return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
5502 return ERR_PTR(-EPROTO);
5505 /* This value should be large enough to cover a tagged ethernet header plus
5506 * maximally sized IP and TCP or UDP headers.
5508 #define MAX_IP_HDR_LEN 128
5510 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
5519 err = skb_maybe_pull_tail(skb,
5520 sizeof(struct iphdr),
5525 if (ip_is_fragment(ip_hdr(skb)))
5528 off = ip_hdrlen(skb);
5535 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
5537 return PTR_ERR(csum);
5540 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
5543 ip_hdr(skb)->protocol, 0);
5550 /* This value should be large enough to cover a tagged ethernet header plus
5551 * an IPv6 header, all options, and a maximal TCP or UDP header.
5553 #define MAX_IPV6_HDR_LEN 256
5555 #define OPT_HDR(type, skb, off) \
5556 (type *)(skb_network_header(skb) + (off))
5558 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
5571 off = sizeof(struct ipv6hdr);
5573 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
5577 nexthdr = ipv6_hdr(skb)->nexthdr;
5579 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
5580 while (off <= len && !done) {
5582 case IPPROTO_DSTOPTS:
5583 case IPPROTO_HOPOPTS:
5584 case IPPROTO_ROUTING: {
5585 struct ipv6_opt_hdr *hp;
5587 err = skb_maybe_pull_tail(skb,
5589 sizeof(struct ipv6_opt_hdr),
5594 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
5595 nexthdr = hp->nexthdr;
5596 off += ipv6_optlen(hp);
5600 struct ip_auth_hdr *hp;
5602 err = skb_maybe_pull_tail(skb,
5604 sizeof(struct ip_auth_hdr),
5609 hp = OPT_HDR(struct ip_auth_hdr, skb, off);
5610 nexthdr = hp->nexthdr;
5611 off += ipv6_authlen(hp);
5614 case IPPROTO_FRAGMENT: {
5615 struct frag_hdr *hp;
5617 err = skb_maybe_pull_tail(skb,
5619 sizeof(struct frag_hdr),
5624 hp = OPT_HDR(struct frag_hdr, skb, off);
5626 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
5629 nexthdr = hp->nexthdr;
5630 off += sizeof(struct frag_hdr);
5641 if (!done || fragment)
5644 csum = skb_checksum_setup_ip(skb, nexthdr, off);
5646 return PTR_ERR(csum);
5649 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
5650 &ipv6_hdr(skb)->daddr,
5651 skb->len - off, nexthdr, 0);
5659 * skb_checksum_setup - set up partial checksum offset
5660 * @skb: the skb to set up
5661 * @recalculate: if true the pseudo-header checksum will be recalculated
5663 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
5667 switch (skb->protocol) {
5668 case htons(ETH_P_IP):
5669 err = skb_checksum_setup_ipv4(skb, recalculate);
5672 case htons(ETH_P_IPV6):
5673 err = skb_checksum_setup_ipv6(skb, recalculate);
5683 EXPORT_SYMBOL(skb_checksum_setup);
5686 * skb_checksum_maybe_trim - maybe trims the given skb
5687 * @skb: the skb to check
5688 * @transport_len: the data length beyond the network header
5690 * Checks whether the given skb has data beyond the given transport length.
5691 * If so, returns a cloned skb trimmed to this transport length.
5692 * Otherwise returns the provided skb. Returns NULL in error cases
5693 * (e.g. transport_len exceeds skb length or out-of-memory).
5695 * Caller needs to set the skb transport header and free any returned skb if it
5696 * differs from the provided skb.
5698 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
5699 unsigned int transport_len)
5701 struct sk_buff *skb_chk;
5702 unsigned int len = skb_transport_offset(skb) + transport_len;
5707 else if (skb->len == len)
5710 skb_chk = skb_clone(skb, GFP_ATOMIC);
5714 ret = pskb_trim_rcsum(skb_chk, len);
5724 * skb_checksum_trimmed - validate checksum of an skb
5725 * @skb: the skb to check
5726 * @transport_len: the data length beyond the network header
5727 * @skb_chkf: checksum function to use
5729 * Applies the given checksum function skb_chkf to the provided skb.
5730 * Returns a checked and maybe trimmed skb. Returns NULL on error.
5732 * If the skb has data beyond the given transport length, then a
5733 * trimmed & cloned skb is checked and returned.
5735 * Caller needs to set the skb transport header and free any returned skb if it
5736 * differs from the provided skb.
5738 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
5739 unsigned int transport_len,
5740 __sum16(*skb_chkf)(struct sk_buff *skb))
5742 struct sk_buff *skb_chk;
5743 unsigned int offset = skb_transport_offset(skb);
5746 skb_chk = skb_checksum_maybe_trim(skb, transport_len);
5750 if (!pskb_may_pull(skb_chk, offset))
5753 skb_pull_rcsum(skb_chk, offset);
5754 ret = skb_chkf(skb_chk);
5755 skb_push_rcsum(skb_chk, offset);
5763 if (skb_chk && skb_chk != skb)
5769 EXPORT_SYMBOL(skb_checksum_trimmed);
5771 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
5773 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
5776 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
5778 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
5781 skb_release_head_state(skb);
5782 kmem_cache_free(skbuff_cache, skb);
5787 EXPORT_SYMBOL(kfree_skb_partial);
5790 * skb_try_coalesce - try to merge skb to prior one
5792 * @from: buffer to add
5793 * @fragstolen: pointer to boolean
5794 * @delta_truesize: how much more was allocated than was requested
5796 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
5797 bool *fragstolen, int *delta_truesize)
5799 struct skb_shared_info *to_shinfo, *from_shinfo;
5800 int i, delta, len = from->len;
5802 *fragstolen = false;
5807 /* In general, avoid mixing page_pool and non-page_pool allocated
5808 * pages within the same SKB. In theory we could take full
5809 * references if @from is cloned and !@to->pp_recycle but its
5810 * tricky (due to potential race with the clone disappearing) and
5811 * rare, so not worth dealing with.
5813 if (to->pp_recycle != from->pp_recycle)
5816 if (len <= skb_tailroom(to)) {
5818 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
5819 *delta_truesize = 0;
5823 to_shinfo = skb_shinfo(to);
5824 from_shinfo = skb_shinfo(from);
5825 if (to_shinfo->frag_list || from_shinfo->frag_list)
5827 if (skb_zcopy(to) || skb_zcopy(from))
5830 if (skb_headlen(from) != 0) {
5832 unsigned int offset;
5834 if (to_shinfo->nr_frags +
5835 from_shinfo->nr_frags >= MAX_SKB_FRAGS)
5838 if (skb_head_is_locked(from))
5841 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
5843 page = virt_to_head_page(from->head);
5844 offset = from->data - (unsigned char *)page_address(page);
5846 skb_fill_page_desc(to, to_shinfo->nr_frags,
5847 page, offset, skb_headlen(from));
5850 if (to_shinfo->nr_frags +
5851 from_shinfo->nr_frags > MAX_SKB_FRAGS)
5854 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
5857 WARN_ON_ONCE(delta < len);
5859 memcpy(to_shinfo->frags + to_shinfo->nr_frags,
5861 from_shinfo->nr_frags * sizeof(skb_frag_t));
5862 to_shinfo->nr_frags += from_shinfo->nr_frags;
5864 if (!skb_cloned(from))
5865 from_shinfo->nr_frags = 0;
5867 /* if the skb is not cloned this does nothing
5868 * since we set nr_frags to 0.
5870 if (skb_pp_frag_ref(from)) {
5871 for (i = 0; i < from_shinfo->nr_frags; i++)
5872 __skb_frag_ref(&from_shinfo->frags[i]);
5875 to->truesize += delta;
5877 to->data_len += len;
5879 *delta_truesize = delta;
5882 EXPORT_SYMBOL(skb_try_coalesce);
5885 * skb_scrub_packet - scrub an skb
5887 * @skb: buffer to clean
5888 * @xnet: packet is crossing netns
5890 * skb_scrub_packet can be used after encapsulating or decapsulting a packet
5891 * into/from a tunnel. Some information have to be cleared during these
5893 * skb_scrub_packet can also be used to clean a skb before injecting it in
5894 * another namespace (@xnet == true). We have to clear all information in the
5895 * skb that could impact namespace isolation.
5897 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
5899 skb->pkt_type = PACKET_HOST;
5905 nf_reset_trace(skb);
5907 #ifdef CONFIG_NET_SWITCHDEV
5908 skb->offload_fwd_mark = 0;
5909 skb->offload_l3_fwd_mark = 0;
5917 skb_clear_tstamp(skb);
5919 EXPORT_SYMBOL_GPL(skb_scrub_packet);
5921 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
5923 int mac_len, meta_len;
5926 if (skb_cow(skb, skb_headroom(skb)) < 0) {
5931 mac_len = skb->data - skb_mac_header(skb);
5932 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
5933 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
5934 mac_len - VLAN_HLEN - ETH_TLEN);
5937 meta_len = skb_metadata_len(skb);
5939 meta = skb_metadata_end(skb) - meta_len;
5940 memmove(meta + VLAN_HLEN, meta, meta_len);
5943 skb->mac_header += VLAN_HLEN;
5947 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
5949 struct vlan_hdr *vhdr;
5952 if (unlikely(skb_vlan_tag_present(skb))) {
5953 /* vlan_tci is already set-up so leave this for another time */
5957 skb = skb_share_check(skb, GFP_ATOMIC);
5960 /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */
5961 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short))))
5964 vhdr = (struct vlan_hdr *)skb->data;
5965 vlan_tci = ntohs(vhdr->h_vlan_TCI);
5966 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
5968 skb_pull_rcsum(skb, VLAN_HLEN);
5969 vlan_set_encap_proto(skb, vhdr);
5971 skb = skb_reorder_vlan_header(skb);
5975 skb_reset_network_header(skb);
5976 if (!skb_transport_header_was_set(skb))
5977 skb_reset_transport_header(skb);
5978 skb_reset_mac_len(skb);
5986 EXPORT_SYMBOL(skb_vlan_untag);
5988 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len)
5990 if (!pskb_may_pull(skb, write_len))
5993 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
5996 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5998 EXPORT_SYMBOL(skb_ensure_writable);
6000 int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev)
6002 int needed_headroom = dev->needed_headroom;
6003 int needed_tailroom = dev->needed_tailroom;
6005 /* For tail taggers, we need to pad short frames ourselves, to ensure
6006 * that the tail tag does not fail at its role of being at the end of
6007 * the packet, once the conduit interface pads the frame. Account for
6008 * that pad length here, and pad later.
6010 if (unlikely(needed_tailroom && skb->len < ETH_ZLEN))
6011 needed_tailroom += ETH_ZLEN - skb->len;
6012 /* skb_headroom() returns unsigned int... */
6013 needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0);
6014 needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0);
6016 if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb)))
6017 /* No reallocation needed, yay! */
6020 return pskb_expand_head(skb, needed_headroom, needed_tailroom,
6023 EXPORT_SYMBOL(skb_ensure_writable_head_tail);
6025 /* remove VLAN header from packet and update csum accordingly.
6026 * expects a non skb_vlan_tag_present skb with a vlan tag payload
6028 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
6030 int offset = skb->data - skb_mac_header(skb);
6033 if (WARN_ONCE(offset,
6034 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
6039 err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
6043 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
6045 vlan_remove_tag(skb, vlan_tci);
6047 skb->mac_header += VLAN_HLEN;
6049 if (skb_network_offset(skb) < ETH_HLEN)
6050 skb_set_network_header(skb, ETH_HLEN);
6052 skb_reset_mac_len(skb);
6056 EXPORT_SYMBOL(__skb_vlan_pop);
6058 /* Pop a vlan tag either from hwaccel or from payload.
6059 * Expects skb->data at mac header.
6061 int skb_vlan_pop(struct sk_buff *skb)
6067 if (likely(skb_vlan_tag_present(skb))) {
6068 __vlan_hwaccel_clear_tag(skb);
6070 if (unlikely(!eth_type_vlan(skb->protocol)))
6073 err = __skb_vlan_pop(skb, &vlan_tci);
6077 /* move next vlan tag to hw accel tag */
6078 if (likely(!eth_type_vlan(skb->protocol)))
6081 vlan_proto = skb->protocol;
6082 err = __skb_vlan_pop(skb, &vlan_tci);
6086 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
6089 EXPORT_SYMBOL(skb_vlan_pop);
6091 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
6092 * Expects skb->data at mac header.
6094 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
6096 if (skb_vlan_tag_present(skb)) {
6097 int offset = skb->data - skb_mac_header(skb);
6100 if (WARN_ONCE(offset,
6101 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
6106 err = __vlan_insert_tag(skb, skb->vlan_proto,
6107 skb_vlan_tag_get(skb));
6111 skb->protocol = skb->vlan_proto;
6112 skb->mac_len += VLAN_HLEN;
6114 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
6116 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
6119 EXPORT_SYMBOL(skb_vlan_push);
6122 * skb_eth_pop() - Drop the Ethernet header at the head of a packet
6124 * @skb: Socket buffer to modify
6126 * Drop the Ethernet header of @skb.
6128 * Expects that skb->data points to the mac header and that no VLAN tags are
6131 * Returns 0 on success, -errno otherwise.
6133 int skb_eth_pop(struct sk_buff *skb)
6135 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) ||
6136 skb_network_offset(skb) < ETH_HLEN)
6139 skb_pull_rcsum(skb, ETH_HLEN);
6140 skb_reset_mac_header(skb);
6141 skb_reset_mac_len(skb);
6145 EXPORT_SYMBOL(skb_eth_pop);
6148 * skb_eth_push() - Add a new Ethernet header at the head of a packet
6150 * @skb: Socket buffer to modify
6151 * @dst: Destination MAC address of the new header
6152 * @src: Source MAC address of the new header
6154 * Prepend @skb with a new Ethernet header.
6156 * Expects that skb->data points to the mac header, which must be empty.
6158 * Returns 0 on success, -errno otherwise.
6160 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
6161 const unsigned char *src)
6166 if (skb_network_offset(skb) || skb_vlan_tag_present(skb))
6169 err = skb_cow_head(skb, sizeof(*eth));
6173 skb_push(skb, sizeof(*eth));
6174 skb_reset_mac_header(skb);
6175 skb_reset_mac_len(skb);
6178 ether_addr_copy(eth->h_dest, dst);
6179 ether_addr_copy(eth->h_source, src);
6180 eth->h_proto = skb->protocol;
6182 skb_postpush_rcsum(skb, eth, sizeof(*eth));
6186 EXPORT_SYMBOL(skb_eth_push);
6188 /* Update the ethertype of hdr and the skb csum value if required. */
6189 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
6192 if (skb->ip_summed == CHECKSUM_COMPLETE) {
6193 __be16 diff[] = { ~hdr->h_proto, ethertype };
6195 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
6198 hdr->h_proto = ethertype;
6202 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
6206 * @mpls_lse: MPLS label stack entry to push
6207 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
6208 * @mac_len: length of the MAC header
6209 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
6212 * Expects skb->data at mac header.
6214 * Returns 0 on success, -errno otherwise.
6216 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
6217 int mac_len, bool ethernet)
6219 struct mpls_shim_hdr *lse;
6222 if (unlikely(!eth_p_mpls(mpls_proto)))
6225 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
6226 if (skb->encapsulation)
6229 err = skb_cow_head(skb, MPLS_HLEN);
6233 if (!skb->inner_protocol) {
6234 skb_set_inner_network_header(skb, skb_network_offset(skb));
6235 skb_set_inner_protocol(skb, skb->protocol);
6238 skb_push(skb, MPLS_HLEN);
6239 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
6241 skb_reset_mac_header(skb);
6242 skb_set_network_header(skb, mac_len);
6243 skb_reset_mac_len(skb);
6245 lse = mpls_hdr(skb);
6246 lse->label_stack_entry = mpls_lse;
6247 skb_postpush_rcsum(skb, lse, MPLS_HLEN);
6249 if (ethernet && mac_len >= ETH_HLEN)
6250 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
6251 skb->protocol = mpls_proto;
6255 EXPORT_SYMBOL_GPL(skb_mpls_push);
6258 * skb_mpls_pop() - pop the outermost MPLS header
6261 * @next_proto: ethertype of header after popped MPLS header
6262 * @mac_len: length of the MAC header
6263 * @ethernet: flag to indicate if the packet is ethernet
6265 * Expects skb->data at mac header.
6267 * Returns 0 on success, -errno otherwise.
6269 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
6274 if (unlikely(!eth_p_mpls(skb->protocol)))
6277 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
6281 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
6282 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
6285 __skb_pull(skb, MPLS_HLEN);
6286 skb_reset_mac_header(skb);
6287 skb_set_network_header(skb, mac_len);
6289 if (ethernet && mac_len >= ETH_HLEN) {
6292 /* use mpls_hdr() to get ethertype to account for VLANs. */
6293 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
6294 skb_mod_eth_type(skb, hdr, next_proto);
6296 skb->protocol = next_proto;
6300 EXPORT_SYMBOL_GPL(skb_mpls_pop);
6303 * skb_mpls_update_lse() - modify outermost MPLS header and update csum
6306 * @mpls_lse: new MPLS label stack entry to update to
6308 * Expects skb->data at mac header.
6310 * Returns 0 on success, -errno otherwise.
6312 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
6316 if (unlikely(!eth_p_mpls(skb->protocol)))
6319 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
6323 if (skb->ip_summed == CHECKSUM_COMPLETE) {
6324 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
6326 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
6329 mpls_hdr(skb)->label_stack_entry = mpls_lse;
6333 EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
6336 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
6340 * Expects skb->data at mac header.
6342 * Returns 0 on success, -errno otherwise.
6344 int skb_mpls_dec_ttl(struct sk_buff *skb)
6349 if (unlikely(!eth_p_mpls(skb->protocol)))
6352 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
6355 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
6356 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
6360 lse &= ~MPLS_LS_TTL_MASK;
6361 lse |= ttl << MPLS_LS_TTL_SHIFT;
6363 return skb_mpls_update_lse(skb, cpu_to_be32(lse));
6365 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
6368 * alloc_skb_with_frags - allocate skb with page frags
6370 * @header_len: size of linear part
6371 * @data_len: needed length in frags
6372 * @order: max page order desired.
6373 * @errcode: pointer to error code if any
6374 * @gfp_mask: allocation mask
6376 * This can be used to allocate a paged skb, given a maximal order for frags.
6378 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
6379 unsigned long data_len,
6384 unsigned long chunk;
6385 struct sk_buff *skb;
6389 *errcode = -EMSGSIZE;
6390 if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order)))
6393 *errcode = -ENOBUFS;
6394 skb = alloc_skb(header_len, gfp_mask);
6399 if (nr_frags == MAX_SKB_FRAGS - 1)
6401 while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order))
6405 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
6414 page = alloc_page(gfp_mask);
6418 chunk = min_t(unsigned long, data_len,
6419 PAGE_SIZE << order);
6420 skb_fill_page_desc(skb, nr_frags, page, 0, chunk);
6422 skb->truesize += (PAGE_SIZE << order);
6431 EXPORT_SYMBOL(alloc_skb_with_frags);
6433 /* carve out the first off bytes from skb when off < headlen */
6434 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
6435 const int headlen, gfp_t gfp_mask)
6438 unsigned int size = skb_end_offset(skb);
6439 int new_hlen = headlen - off;
6442 if (skb_pfmemalloc(skb))
6443 gfp_mask |= __GFP_MEMALLOC;
6445 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
6448 size = SKB_WITH_OVERHEAD(size);
6450 /* Copy real data, and all frags */
6451 skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
6454 memcpy((struct skb_shared_info *)(data + size),
6456 offsetof(struct skb_shared_info,
6457 frags[skb_shinfo(skb)->nr_frags]));
6458 if (skb_cloned(skb)) {
6459 /* drop the old head gracefully */
6460 if (skb_orphan_frags(skb, gfp_mask)) {
6461 skb_kfree_head(data, size);
6464 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
6465 skb_frag_ref(skb, i);
6466 if (skb_has_frag_list(skb))
6467 skb_clone_fraglist(skb);
6468 skb_release_data(skb, SKB_CONSUMED, false);
6470 /* we can reuse existing recount- all we did was
6473 skb_free_head(skb, false);
6479 skb_set_end_offset(skb, size);
6480 skb_set_tail_pointer(skb, skb_headlen(skb));
6481 skb_headers_offset_update(skb, 0);
6485 atomic_set(&skb_shinfo(skb)->dataref, 1);
6490 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
6492 /* carve out the first eat bytes from skb's frag_list. May recurse into
6495 static int pskb_carve_frag_list(struct sk_buff *skb,
6496 struct skb_shared_info *shinfo, int eat,
6499 struct sk_buff *list = shinfo->frag_list;
6500 struct sk_buff *clone = NULL;
6501 struct sk_buff *insp = NULL;
6505 pr_err("Not enough bytes to eat. Want %d\n", eat);
6508 if (list->len <= eat) {
6509 /* Eaten as whole. */
6514 /* Eaten partially. */
6515 if (skb_shared(list)) {
6516 clone = skb_clone(list, gfp_mask);
6522 /* This may be pulled without problems. */
6525 if (pskb_carve(list, eat, gfp_mask) < 0) {
6533 /* Free pulled out fragments. */
6534 while ((list = shinfo->frag_list) != insp) {
6535 shinfo->frag_list = list->next;
6538 /* And insert new clone at head. */
6541 shinfo->frag_list = clone;
6546 /* carve off first len bytes from skb. Split line (off) is in the
6547 * non-linear part of skb
6549 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
6550 int pos, gfp_t gfp_mask)
6553 unsigned int size = skb_end_offset(skb);
6555 const int nfrags = skb_shinfo(skb)->nr_frags;
6556 struct skb_shared_info *shinfo;
6558 if (skb_pfmemalloc(skb))
6559 gfp_mask |= __GFP_MEMALLOC;
6561 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
6564 size = SKB_WITH_OVERHEAD(size);
6566 memcpy((struct skb_shared_info *)(data + size),
6567 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
6568 if (skb_orphan_frags(skb, gfp_mask)) {
6569 skb_kfree_head(data, size);
6572 shinfo = (struct skb_shared_info *)(data + size);
6573 for (i = 0; i < nfrags; i++) {
6574 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
6576 if (pos + fsize > off) {
6577 shinfo->frags[k] = skb_shinfo(skb)->frags[i];
6581 * We have two variants in this case:
6582 * 1. Move all the frag to the second
6583 * part, if it is possible. F.e.
6584 * this approach is mandatory for TUX,
6585 * where splitting is expensive.
6586 * 2. Split is accurately. We make this.
6588 skb_frag_off_add(&shinfo->frags[0], off - pos);
6589 skb_frag_size_sub(&shinfo->frags[0], off - pos);
6591 skb_frag_ref(skb, i);
6596 shinfo->nr_frags = k;
6597 if (skb_has_frag_list(skb))
6598 skb_clone_fraglist(skb);
6600 /* split line is in frag list */
6601 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) {
6602 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */
6603 if (skb_has_frag_list(skb))
6604 kfree_skb_list(skb_shinfo(skb)->frag_list);
6605 skb_kfree_head(data, size);
6608 skb_release_data(skb, SKB_CONSUMED, false);
6613 skb_set_end_offset(skb, size);
6614 skb_reset_tail_pointer(skb);
6615 skb_headers_offset_update(skb, 0);
6620 skb->data_len = skb->len;
6621 atomic_set(&skb_shinfo(skb)->dataref, 1);
6625 /* remove len bytes from the beginning of the skb */
6626 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
6628 int headlen = skb_headlen(skb);
6631 return pskb_carve_inside_header(skb, len, headlen, gfp);
6633 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
6636 /* Extract to_copy bytes starting at off from skb, and return this in
6639 struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
6640 int to_copy, gfp_t gfp)
6642 struct sk_buff *clone = skb_clone(skb, gfp);
6647 if (pskb_carve(clone, off, gfp) < 0 ||
6648 pskb_trim(clone, to_copy)) {
6654 EXPORT_SYMBOL(pskb_extract);
6657 * skb_condense - try to get rid of fragments/frag_list if possible
6660 * Can be used to save memory before skb is added to a busy queue.
6661 * If packet has bytes in frags and enough tail room in skb->head,
6662 * pull all of them, so that we can free the frags right now and adjust
6665 * We do not reallocate skb->head thus can not fail.
6666 * Caller must re-evaluate skb->truesize if needed.
6668 void skb_condense(struct sk_buff *skb)
6670 if (skb->data_len) {
6671 if (skb->data_len > skb->end - skb->tail ||
6675 /* Nice, we can free page frag(s) right now */
6676 __pskb_pull_tail(skb, skb->data_len);
6678 /* At this point, skb->truesize might be over estimated,
6679 * because skb had a fragment, and fragments do not tell
6681 * When we pulled its content into skb->head, fragment
6682 * was freed, but __pskb_pull_tail() could not possibly
6683 * adjust skb->truesize, not knowing the frag truesize.
6685 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
6687 EXPORT_SYMBOL(skb_condense);
6689 #ifdef CONFIG_SKB_EXTENSIONS
6690 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
6692 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
6696 * __skb_ext_alloc - allocate a new skb extensions storage
6698 * @flags: See kmalloc().
6700 * Returns the newly allocated pointer. The pointer can later attached to a
6701 * skb via __skb_ext_set().
6702 * Note: caller must handle the skb_ext as an opaque data.
6704 struct skb_ext *__skb_ext_alloc(gfp_t flags)
6706 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
6709 memset(new->offset, 0, sizeof(new->offset));
6710 refcount_set(&new->refcnt, 1);
6716 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
6717 unsigned int old_active)
6719 struct skb_ext *new;
6721 if (refcount_read(&old->refcnt) == 1)
6724 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
6728 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
6729 refcount_set(&new->refcnt, 1);
6732 if (old_active & (1 << SKB_EXT_SEC_PATH)) {
6733 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
6736 for (i = 0; i < sp->len; i++)
6737 xfrm_state_hold(sp->xvec[i]);
6740 #ifdef CONFIG_MCTP_FLOWS
6741 if (old_active & (1 << SKB_EXT_MCTP)) {
6742 struct mctp_flow *flow = skb_ext_get_ptr(old, SKB_EXT_MCTP);
6745 refcount_inc(&flow->key->refs);
6753 * __skb_ext_set - attach the specified extension storage to this skb
6756 * @ext: extension storage previously allocated via __skb_ext_alloc()
6758 * Existing extensions, if any, are cleared.
6760 * Returns the pointer to the extension.
6762 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
6763 struct skb_ext *ext)
6765 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
6768 newlen = newoff + skb_ext_type_len[id];
6769 ext->chunks = newlen;
6770 ext->offset[id] = newoff;
6771 skb->extensions = ext;
6772 skb->active_extensions = 1 << id;
6773 return skb_ext_get_ptr(ext, id);
6777 * skb_ext_add - allocate space for given extension, COW if needed
6779 * @id: extension to allocate space for
6781 * Allocates enough space for the given extension.
6782 * If the extension is already present, a pointer to that extension
6785 * If the skb was cloned, COW applies and the returned memory can be
6786 * modified without changing the extension space of clones buffers.
6788 * Returns pointer to the extension or NULL on allocation failure.
6790 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
6792 struct skb_ext *new, *old = NULL;
6793 unsigned int newlen, newoff;
6795 if (skb->active_extensions) {
6796 old = skb->extensions;
6798 new = skb_ext_maybe_cow(old, skb->active_extensions);
6802 if (__skb_ext_exist(new, id))
6805 newoff = new->chunks;
6807 newoff = SKB_EXT_CHUNKSIZEOF(*new);
6809 new = __skb_ext_alloc(GFP_ATOMIC);
6814 newlen = newoff + skb_ext_type_len[id];
6815 new->chunks = newlen;
6816 new->offset[id] = newoff;
6819 skb->extensions = new;
6820 skb->active_extensions |= 1 << id;
6821 return skb_ext_get_ptr(new, id);
6823 EXPORT_SYMBOL(skb_ext_add);
6826 static void skb_ext_put_sp(struct sec_path *sp)
6830 for (i = 0; i < sp->len; i++)
6831 xfrm_state_put(sp->xvec[i]);
6835 #ifdef CONFIG_MCTP_FLOWS
6836 static void skb_ext_put_mctp(struct mctp_flow *flow)
6839 mctp_key_unref(flow->key);
6843 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
6845 struct skb_ext *ext = skb->extensions;
6847 skb->active_extensions &= ~(1 << id);
6848 if (skb->active_extensions == 0) {
6849 skb->extensions = NULL;
6852 } else if (id == SKB_EXT_SEC_PATH &&
6853 refcount_read(&ext->refcnt) == 1) {
6854 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
6861 EXPORT_SYMBOL(__skb_ext_del);
6863 void __skb_ext_put(struct skb_ext *ext)
6865 /* If this is last clone, nothing can increment
6866 * it after check passes. Avoids one atomic op.
6868 if (refcount_read(&ext->refcnt) == 1)
6871 if (!refcount_dec_and_test(&ext->refcnt))
6875 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
6876 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
6878 #ifdef CONFIG_MCTP_FLOWS
6879 if (__skb_ext_exist(ext, SKB_EXT_MCTP))
6880 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP));
6883 kmem_cache_free(skbuff_ext_cache, ext);
6885 EXPORT_SYMBOL(__skb_ext_put);
6886 #endif /* CONFIG_SKB_EXTENSIONS */
6889 * skb_attempt_defer_free - queue skb for remote freeing
6892 * Put @skb in a per-cpu list, using the cpu which
6893 * allocated the skb/pages to reduce false sharing
6894 * and memory zone spinlock contention.
6896 void skb_attempt_defer_free(struct sk_buff *skb)
6898 int cpu = skb->alloc_cpu;
6899 struct softnet_data *sd;
6900 unsigned int defer_max;
6903 if (WARN_ON_ONCE(cpu >= nr_cpu_ids) ||
6905 cpu == raw_smp_processor_id()) {
6906 nodefer: __kfree_skb(skb);
6910 DEBUG_NET_WARN_ON_ONCE(skb_dst(skb));
6911 DEBUG_NET_WARN_ON_ONCE(skb->destructor);
6913 sd = &per_cpu(softnet_data, cpu);
6914 defer_max = READ_ONCE(sysctl_skb_defer_max);
6915 if (READ_ONCE(sd->defer_count) >= defer_max)
6918 spin_lock_bh(&sd->defer_lock);
6919 /* Send an IPI every time queue reaches half capacity. */
6920 kick = sd->defer_count == (defer_max >> 1);
6921 /* Paired with the READ_ONCE() few lines above */
6922 WRITE_ONCE(sd->defer_count, sd->defer_count + 1);
6924 skb->next = sd->defer_list;
6925 /* Paired with READ_ONCE() in skb_defer_free_flush() */
6926 WRITE_ONCE(sd->defer_list, skb);
6927 spin_unlock_bh(&sd->defer_lock);
6929 /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU
6930 * if we are unlucky enough (this seems very unlikely).
6932 if (unlikely(kick) && !cmpxchg(&sd->defer_ipi_scheduled, 0, 1))
6933 smp_call_function_single_async(cpu, &sd->defer_csd);
6936 static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,
6937 size_t offset, size_t len)
6942 kaddr = kmap_local_page(page);
6943 csum = csum_partial(kaddr + offset, len, 0);
6944 kunmap_local(kaddr);
6945 skb->csum = csum_block_add(skb->csum, csum, skb->len);
6949 * skb_splice_from_iter - Splice (or copy) pages to skbuff
6950 * @skb: The buffer to add pages to
6951 * @iter: Iterator representing the pages to be added
6952 * @maxsize: Maximum amount of pages to be added
6953 * @gfp: Allocation flags
6955 * This is a common helper function for supporting MSG_SPLICE_PAGES. It
6956 * extracts pages from an iterator and adds them to the socket buffer if
6957 * possible, copying them to fragments if not possible (such as if they're slab
6960 * Returns the amount of data spliced/copied or -EMSGSIZE if there's
6961 * insufficient space in the buffer to transfer anything.
6963 ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
6964 ssize_t maxsize, gfp_t gfp)
6966 size_t frag_limit = READ_ONCE(sysctl_max_skb_frags);
6967 struct page *pages[8], **ppages = pages;
6968 ssize_t spliced = 0, ret = 0;
6971 while (iter->count > 0) {
6972 ssize_t space, nr, len;
6976 space = frag_limit - skb_shinfo(skb)->nr_frags;
6980 /* We might be able to coalesce without increasing nr_frags */
6981 nr = clamp_t(size_t, space, 1, ARRAY_SIZE(pages));
6983 len = iov_iter_extract_pages(iter, &ppages, maxsize, nr, 0, &off);
6991 struct page *page = pages[i++];
6992 size_t part = min_t(size_t, PAGE_SIZE - off, len);
6995 if (WARN_ON_ONCE(!sendpage_ok(page)))
6998 ret = skb_append_pagefrags(skb, page, off, part,
7001 iov_iter_revert(iter, len);
7005 if (skb->ip_summed == CHECKSUM_NONE)
7006 skb_splice_csum_page(skb, page, off, part);
7019 skb_len_add(skb, spliced);
7020 return spliced ?: ret;
7022 EXPORT_SYMBOL(skb_splice_from_iter);
7024 static __always_inline
7025 size_t memcpy_from_iter_csum(void *iter_from, size_t progress,
7026 size_t len, void *to, void *priv2)
7028 __wsum *csum = priv2;
7029 __wsum next = csum_partial_copy_nocheck(iter_from, to + progress, len);
7031 *csum = csum_block_add(*csum, next, progress);
7035 static __always_inline
7036 size_t copy_from_user_iter_csum(void __user *iter_from, size_t progress,
7037 size_t len, void *to, void *priv2)
7039 __wsum next, *csum = priv2;
7041 next = csum_and_copy_from_user(iter_from, to + progress, len);
7042 *csum = csum_block_add(*csum, next, progress);
7043 return next ? 0 : len;
7046 bool csum_and_copy_from_iter_full(void *addr, size_t bytes,
7047 __wsum *csum, struct iov_iter *i)
7051 if (WARN_ON_ONCE(!i->data_source))
7053 copied = iterate_and_advance2(i, bytes, addr, csum,
7054 copy_from_user_iter_csum,
7055 memcpy_from_iter_csum);
7056 if (likely(copied == bytes))
7058 iov_iter_revert(i, copied);
7061 EXPORT_SYMBOL(csum_and_copy_from_iter_full);