1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * IPv6 output functions
4 * Linux INET6 implementation
7 * Pedro Roque <roque@di.fc.ul.pt>
9 * Based on linux/net/ipv4/ip_output.c
12 * A.N.Kuznetsov : airthmetics in fragmentation.
13 * extension headers are implemented.
14 * route changes now work.
15 * ip6_forward does not confuse sniffers.
18 * H. von Brand : Added missing #include <linux/string.h>
19 * Imran Patel : frag id should be in NBO
20 * Kazunori MIYAZAWA @USAGI
21 * : add ip6_append_data and related functions
25 #include <linux/errno.h>
26 #include <linux/kernel.h>
27 #include <linux/string.h>
28 #include <linux/socket.h>
29 #include <linux/net.h>
30 #include <linux/netdevice.h>
31 #include <linux/if_arp.h>
32 #include <linux/in6.h>
33 #include <linux/tcp.h>
34 #include <linux/route.h>
35 #include <linux/module.h>
36 #include <linux/slab.h>
38 #include <linux/bpf-cgroup.h>
39 #include <linux/netfilter.h>
40 #include <linux/netfilter_ipv6.h>
47 #include <net/ndisc.h>
48 #include <net/protocol.h>
49 #include <net/ip6_route.h>
50 #include <net/addrconf.h>
51 #include <net/rawv6.h>
54 #include <net/checksum.h>
55 #include <linux/mroute6.h>
56 #include <net/l3mdev.h>
57 #include <net/lwtunnel.h>
58 #include <net/ip_tunnels.h>
60 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
62 struct dst_entry *dst = skb_dst(skb);
63 struct net_device *dev = dst->dev;
64 struct inet6_dev *idev = ip6_dst_idev(dst);
65 unsigned int hh_len = LL_RESERVED_SPACE(dev);
66 const struct in6_addr *daddr, *nexthop;
68 struct neighbour *neigh;
71 /* Be paranoid, rather than too clever. */
72 if (unlikely(hh_len > skb_headroom(skb)) && dev->header_ops) {
73 skb = skb_expand_head(skb, hh_len);
75 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
82 if (ipv6_addr_is_multicast(daddr)) {
83 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) &&
84 ((mroute6_is_socket(net, skb) &&
85 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
86 ipv6_chk_mcast_addr(dev, daddr, &hdr->saddr))) {
87 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
89 /* Do not check for IFF_ALLMULTI; multicast routing
90 is not supported in any case.
93 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING,
94 net, sk, newskb, NULL, newskb->dev,
97 if (hdr->hop_limit == 0) {
98 IP6_INC_STATS(net, idev,
99 IPSTATS_MIB_OUTDISCARDS);
105 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len);
106 if (IPV6_ADDR_MC_SCOPE(daddr) <= IPV6_ADDR_SCOPE_NODELOCAL &&
107 !(dev->flags & IFF_LOOPBACK)) {
113 if (lwtunnel_xmit_redirect(dst->lwtstate)) {
114 int res = lwtunnel_xmit(skb);
116 if (res != LWTUNNEL_XMIT_CONTINUE)
120 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUT, skb->len);
123 nexthop = rt6_nexthop((struct rt6_info *)dst, daddr);
124 neigh = __ipv6_neigh_lookup_noref(dev, nexthop);
126 if (unlikely(IS_ERR_OR_NULL(neigh))) {
127 if (unlikely(!neigh))
128 neigh = __neigh_create(&nd_tbl, nexthop, dev, false);
131 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTNOROUTES);
132 kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_CREATEFAIL);
136 sock_confirm_neigh(skb, neigh);
137 ret = neigh_output(neigh, skb, false);
143 ip6_finish_output_gso_slowpath_drop(struct net *net, struct sock *sk,
144 struct sk_buff *skb, unsigned int mtu)
146 struct sk_buff *segs, *nskb;
147 netdev_features_t features;
150 /* Please see corresponding comment in ip_finish_output_gso
151 * describing the cases where GSO segment length exceeds the
154 features = netif_skb_features(skb);
155 segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
156 if (IS_ERR_OR_NULL(segs)) {
163 skb_list_walk_safe(segs, segs, nskb) {
166 skb_mark_not_on_list(segs);
167 /* Last GSO segment can be smaller than gso_size (and MTU).
168 * Adding a fragment header would produce an "atomic fragment",
169 * which is considered harmful (RFC-8021). Avoid that.
171 err = segs->len > mtu ?
172 ip6_fragment(net, sk, segs, ip6_finish_output2) :
173 ip6_finish_output2(net, sk, segs);
181 static int ip6_finish_output_gso(struct net *net, struct sock *sk,
182 struct sk_buff *skb, unsigned int mtu)
184 if (!(IP6CB(skb)->flags & IP6SKB_FAKEJUMBO) &&
185 !skb_gso_validate_network_len(skb, mtu))
186 return ip6_finish_output_gso_slowpath_drop(net, sk, skb, mtu);
188 return ip6_finish_output2(net, sk, skb);
191 static int __ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
195 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
196 /* Policy lookup after SNAT yielded a new policy */
197 if (skb_dst(skb)->xfrm) {
198 IP6CB(skb)->flags |= IP6SKB_REROUTED;
199 return dst_output(net, sk, skb);
203 mtu = ip6_skb_dst_mtu(skb);
205 return ip6_finish_output_gso(net, sk, skb, mtu);
207 if (skb->len > mtu ||
208 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))
209 return ip6_fragment(net, sk, skb, ip6_finish_output2);
211 return ip6_finish_output2(net, sk, skb);
214 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
218 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb);
220 case NET_XMIT_SUCCESS:
222 return __ip6_finish_output(net, sk, skb) ? : ret;
224 kfree_skb_reason(skb, SKB_DROP_REASON_BPF_CGROUP_EGRESS);
229 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
231 struct net_device *dev = skb_dst(skb)->dev, *indev = skb->dev;
232 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
234 skb->protocol = htons(ETH_P_IPV6);
237 if (unlikely(idev->cnf.disable_ipv6)) {
238 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
239 kfree_skb_reason(skb, SKB_DROP_REASON_IPV6DISABLED);
243 return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING,
244 net, sk, skb, indev, dev,
246 !(IP6CB(skb)->flags & IP6SKB_REROUTED));
248 EXPORT_SYMBOL(ip6_output);
250 bool ip6_autoflowlabel(struct net *net, const struct sock *sk)
252 if (!inet6_test_bit(AUTOFLOWLABEL_SET, sk))
253 return ip6_default_np_autolabel(net);
254 return inet6_test_bit(AUTOFLOWLABEL, sk);
258 * xmit an sk_buff (used by TCP, SCTP and DCCP)
259 * Note : socket lock is not held for SYNACK packets, but might be modified
260 * by calls to skb_set_owner_w() and ipv6_local_error(),
261 * which are using proper atomic operations or spinlocks.
263 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6,
264 __u32 mark, struct ipv6_txoptions *opt, int tclass, u32 priority)
266 struct net *net = sock_net(sk);
267 const struct ipv6_pinfo *np = inet6_sk(sk);
268 struct in6_addr *first_hop = &fl6->daddr;
269 struct dst_entry *dst = skb_dst(skb);
270 struct net_device *dev = dst->dev;
271 struct inet6_dev *idev = ip6_dst_idev(dst);
272 struct hop_jumbo_hdr *hop_jumbo;
273 int hoplen = sizeof(*hop_jumbo);
274 unsigned int head_room;
276 u8 proto = fl6->flowi6_proto;
277 int seg_len = skb->len;
281 head_room = sizeof(struct ipv6hdr) + hoplen + LL_RESERVED_SPACE(dev);
283 head_room += opt->opt_nflen + opt->opt_flen;
285 if (unlikely(head_room > skb_headroom(skb))) {
286 skb = skb_expand_head(skb, head_room);
288 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
294 seg_len += opt->opt_nflen + opt->opt_flen;
297 ipv6_push_frag_opts(skb, opt, &proto);
300 ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop,
304 if (unlikely(seg_len > IPV6_MAXPLEN)) {
305 hop_jumbo = skb_push(skb, hoplen);
307 hop_jumbo->nexthdr = proto;
308 hop_jumbo->hdrlen = 0;
309 hop_jumbo->tlv_type = IPV6_TLV_JUMBO;
310 hop_jumbo->tlv_len = 4;
311 hop_jumbo->jumbo_payload_len = htonl(seg_len + hoplen);
313 proto = IPPROTO_HOPOPTS;
315 IP6CB(skb)->flags |= IP6SKB_FAKEJUMBO;
318 skb_push(skb, sizeof(struct ipv6hdr));
319 skb_reset_network_header(skb);
323 * Fill in the IPv6 header
326 hlimit = READ_ONCE(np->hop_limit);
328 hlimit = ip6_dst_hoplimit(dst);
330 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel,
331 ip6_autoflowlabel(net, sk), fl6));
333 hdr->payload_len = htons(seg_len);
334 hdr->nexthdr = proto;
335 hdr->hop_limit = hlimit;
337 hdr->saddr = fl6->saddr;
338 hdr->daddr = *first_hop;
340 skb->protocol = htons(ETH_P_IPV6);
341 skb->priority = priority;
345 if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) {
346 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTREQUESTS);
348 /* if egress device is enslaved to an L3 master device pass the
349 * skb to its handler for processing
351 skb = l3mdev_ip6_out((struct sock *)sk, skb);
355 /* hooks should never assume socket lock is held.
356 * we promote our socket to non const
358 return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT,
359 net, (struct sock *)sk, skb, NULL, dev,
364 /* ipv6_local_error() does not require socket lock,
365 * we promote our socket to non const
367 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu);
369 IP6_INC_STATS(net, idev, IPSTATS_MIB_FRAGFAILS);
373 EXPORT_SYMBOL(ip6_xmit);
375 static int ip6_call_ra_chain(struct sk_buff *skb, int sel)
377 struct ip6_ra_chain *ra;
378 struct sock *last = NULL;
380 read_lock(&ip6_ra_lock);
381 for (ra = ip6_ra_chain; ra; ra = ra->next) {
382 struct sock *sk = ra->sk;
383 if (sk && ra->sel == sel &&
384 (!sk->sk_bound_dev_if ||
385 sk->sk_bound_dev_if == skb->dev->ifindex)) {
387 if (inet6_test_bit(RTALERT_ISOLATE, sk) &&
388 !net_eq(sock_net(sk), dev_net(skb->dev))) {
392 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
394 rawv6_rcv(last, skb2);
401 rawv6_rcv(last, skb);
402 read_unlock(&ip6_ra_lock);
405 read_unlock(&ip6_ra_lock);
409 static int ip6_forward_proxy_check(struct sk_buff *skb)
411 struct ipv6hdr *hdr = ipv6_hdr(skb);
412 u8 nexthdr = hdr->nexthdr;
416 if (ipv6_ext_hdr(nexthdr)) {
417 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off);
421 offset = sizeof(struct ipv6hdr);
423 if (nexthdr == IPPROTO_ICMPV6) {
424 struct icmp6hdr *icmp6;
426 if (!pskb_may_pull(skb, (skb_network_header(skb) +
427 offset + 1 - skb->data)))
430 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset);
432 switch (icmp6->icmp6_type) {
433 case NDISC_ROUTER_SOLICITATION:
434 case NDISC_ROUTER_ADVERTISEMENT:
435 case NDISC_NEIGHBOUR_SOLICITATION:
436 case NDISC_NEIGHBOUR_ADVERTISEMENT:
438 /* For reaction involving unicast neighbor discovery
439 * message destined to the proxied address, pass it to
449 * The proxying router can't forward traffic sent to a link-local
450 * address, so signal the sender and discard the packet. This
451 * behavior is clarified by the MIPv6 specification.
453 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) {
454 dst_link_failure(skb);
461 static inline int ip6_forward_finish(struct net *net, struct sock *sk,
464 #ifdef CONFIG_NET_SWITCHDEV
465 if (skb->offload_l3_fwd_mark) {
471 skb_clear_tstamp(skb);
472 return dst_output(net, sk, skb);
475 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
480 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */
481 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu)
487 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
493 int ip6_forward(struct sk_buff *skb)
495 struct dst_entry *dst = skb_dst(skb);
496 struct ipv6hdr *hdr = ipv6_hdr(skb);
497 struct inet6_skb_parm *opt = IP6CB(skb);
498 struct net *net = dev_net(dst->dev);
499 struct inet6_dev *idev;
503 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif));
504 if (net->ipv6.devconf_all->forwarding == 0)
507 if (skb->pkt_type != PACKET_HOST)
510 if (unlikely(skb->sk))
513 if (skb_warn_if_lro(skb))
516 if (!net->ipv6.devconf_all->disable_policy &&
517 (!idev || !idev->cnf.disable_policy) &&
518 !xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) {
519 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
523 skb_forward_csum(skb);
526 * We DO NOT make any processing on
527 * RA packets, pushing them to user level AS IS
528 * without ane WARRANTY that application will be able
529 * to interpret them. The reason is that we
530 * cannot make anything clever here.
532 * We are not end-node, so that if packet contains
533 * AH/ESP, we cannot make anything.
534 * Defragmentation also would be mistake, RA packets
535 * cannot be fragmented, because there is no warranty
536 * that different fragments will go along one path. --ANK
538 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) {
539 if (ip6_call_ra_chain(skb, ntohs(opt->ra)))
544 * check and decrement ttl
546 if (hdr->hop_limit <= 1) {
547 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0);
548 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
550 kfree_skb_reason(skb, SKB_DROP_REASON_IP_INHDR);
554 /* XXX: idev->cnf.proxy_ndp? */
555 if (net->ipv6.devconf_all->proxy_ndp &&
556 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) {
557 int proxied = ip6_forward_proxy_check(skb);
559 /* It's tempting to decrease the hop limit
560 * here by 1, as we do at the end of the
563 * But that would be incorrect, as proxying is
564 * not forwarding. The ip6_input function
565 * will handle this packet locally, and it
566 * depends on the hop limit being unchanged.
568 * One example is the NDP hop limit, that
569 * always has to stay 255, but other would be
570 * similar checks around RA packets, where the
571 * user can even change the desired limit.
573 return ip6_input(skb);
574 } else if (proxied < 0) {
575 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
580 if (!xfrm6_route_forward(skb)) {
581 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
582 SKB_DR_SET(reason, XFRM_POLICY);
587 /* IPv6 specs say nothing about it, but it is clear that we cannot
588 send redirects to source routed frames.
589 We don't send redirects to frames decapsulated from IPsec.
591 if (IP6CB(skb)->iif == dst->dev->ifindex &&
592 opt->srcrt == 0 && !skb_sec_path(skb)) {
593 struct in6_addr *target = NULL;
594 struct inet_peer *peer;
598 * incoming and outgoing devices are the same
602 rt = (struct rt6_info *) dst;
603 if (rt->rt6i_flags & RTF_GATEWAY)
604 target = &rt->rt6i_gateway;
606 target = &hdr->daddr;
608 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr, 1);
610 /* Limit redirects both by destination (here)
611 and by source (inside ndisc_send_redirect)
613 if (inet_peer_xrlim_allow(peer, 1*HZ))
614 ndisc_send_redirect(skb, target);
618 int addrtype = ipv6_addr_type(&hdr->saddr);
620 /* This check is security critical. */
621 if (addrtype == IPV6_ADDR_ANY ||
622 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK))
624 if (addrtype & IPV6_ADDR_LINKLOCAL) {
625 icmpv6_send(skb, ICMPV6_DEST_UNREACH,
626 ICMPV6_NOT_NEIGHBOUR, 0);
631 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS);
633 mtu = ip6_dst_mtu_maybe_forward(dst, true);
634 if (mtu < IPV6_MIN_MTU)
637 if (ip6_pkt_too_big(skb, mtu)) {
638 /* Again, force OUTPUT device used as source address */
640 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
641 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INTOOBIGERRORS);
642 __IP6_INC_STATS(net, ip6_dst_idev(dst),
643 IPSTATS_MIB_FRAGFAILS);
644 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG);
648 if (skb_cow(skb, dst->dev->hard_header_len)) {
649 __IP6_INC_STATS(net, ip6_dst_idev(dst),
650 IPSTATS_MIB_OUTDISCARDS);
656 /* Mangling hops number delayed to point after skb COW */
660 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD,
661 net, NULL, skb, skb->dev, dst->dev,
665 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS);
666 SKB_DR_SET(reason, IP_INADDRERRORS);
668 kfree_skb_reason(skb, reason);
672 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from)
674 to->pkt_type = from->pkt_type;
675 to->priority = from->priority;
676 to->protocol = from->protocol;
678 skb_dst_set(to, dst_clone(skb_dst(from)));
680 to->mark = from->mark;
682 skb_copy_hash(to, from);
684 #ifdef CONFIG_NET_SCHED
685 to->tc_index = from->tc_index;
688 skb_ext_copy(to, from);
689 skb_copy_secmark(to, from);
692 int ip6_fraglist_init(struct sk_buff *skb, unsigned int hlen, u8 *prevhdr,
693 u8 nexthdr, __be32 frag_id,
694 struct ip6_fraglist_iter *iter)
696 unsigned int first_len;
700 *prevhdr = NEXTHDR_FRAGMENT;
701 iter->tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC);
705 iter->frag = skb_shinfo(skb)->frag_list;
706 skb_frag_list_init(skb);
710 iter->frag_id = frag_id;
711 iter->nexthdr = nexthdr;
713 __skb_pull(skb, hlen);
714 fh = __skb_push(skb, sizeof(struct frag_hdr));
715 __skb_push(skb, hlen);
716 skb_reset_network_header(skb);
717 memcpy(skb_network_header(skb), iter->tmp_hdr, hlen);
719 fh->nexthdr = nexthdr;
721 fh->frag_off = htons(IP6_MF);
722 fh->identification = frag_id;
724 first_len = skb_pagelen(skb);
725 skb->data_len = first_len - skb_headlen(skb);
726 skb->len = first_len;
727 ipv6_hdr(skb)->payload_len = htons(first_len - sizeof(struct ipv6hdr));
731 EXPORT_SYMBOL(ip6_fraglist_init);
733 void ip6_fraglist_prepare(struct sk_buff *skb,
734 struct ip6_fraglist_iter *iter)
736 struct sk_buff *frag = iter->frag;
737 unsigned int hlen = iter->hlen;
740 frag->ip_summed = CHECKSUM_NONE;
741 skb_reset_transport_header(frag);
742 fh = __skb_push(frag, sizeof(struct frag_hdr));
743 __skb_push(frag, hlen);
744 skb_reset_network_header(frag);
745 memcpy(skb_network_header(frag), iter->tmp_hdr, hlen);
746 iter->offset += skb->len - hlen - sizeof(struct frag_hdr);
747 fh->nexthdr = iter->nexthdr;
749 fh->frag_off = htons(iter->offset);
751 fh->frag_off |= htons(IP6_MF);
752 fh->identification = iter->frag_id;
753 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr));
754 ip6_copy_metadata(frag, skb);
756 EXPORT_SYMBOL(ip6_fraglist_prepare);
758 void ip6_frag_init(struct sk_buff *skb, unsigned int hlen, unsigned int mtu,
759 unsigned short needed_tailroom, int hdr_room, u8 *prevhdr,
760 u8 nexthdr, __be32 frag_id, struct ip6_frag_state *state)
762 state->prevhdr = prevhdr;
763 state->nexthdr = nexthdr;
764 state->frag_id = frag_id;
769 state->left = skb->len - hlen; /* Space per frame */
770 state->ptr = hlen; /* Where to start from */
772 state->hroom = hdr_room;
773 state->troom = needed_tailroom;
777 EXPORT_SYMBOL(ip6_frag_init);
779 struct sk_buff *ip6_frag_next(struct sk_buff *skb, struct ip6_frag_state *state)
781 u8 *prevhdr = state->prevhdr, *fragnexthdr_offset;
782 struct sk_buff *frag;
787 /* IF: it doesn't fit, use 'mtu' - the data space left */
788 if (len > state->mtu)
790 /* IF: we are not sending up to and including the packet end
791 then align the next start on an eight byte boundary */
792 if (len < state->left)
795 /* Allocate buffer */
796 frag = alloc_skb(len + state->hlen + sizeof(struct frag_hdr) +
797 state->hroom + state->troom, GFP_ATOMIC);
799 return ERR_PTR(-ENOMEM);
802 * Set up data on packet
805 ip6_copy_metadata(frag, skb);
806 skb_reserve(frag, state->hroom);
807 skb_put(frag, len + state->hlen + sizeof(struct frag_hdr));
808 skb_reset_network_header(frag);
809 fh = (struct frag_hdr *)(skb_network_header(frag) + state->hlen);
810 frag->transport_header = (frag->network_header + state->hlen +
811 sizeof(struct frag_hdr));
814 * Charge the memory for the fragment to any owner
818 skb_set_owner_w(frag, skb->sk);
821 * Copy the packet header into the new buffer.
823 skb_copy_from_linear_data(skb, skb_network_header(frag), state->hlen);
825 fragnexthdr_offset = skb_network_header(frag);
826 fragnexthdr_offset += prevhdr - skb_network_header(skb);
827 *fragnexthdr_offset = NEXTHDR_FRAGMENT;
830 * Build fragment header.
832 fh->nexthdr = state->nexthdr;
834 fh->identification = state->frag_id;
837 * Copy a block of the IP datagram.
839 BUG_ON(skb_copy_bits(skb, state->ptr, skb_transport_header(frag),
843 fh->frag_off = htons(state->offset);
845 fh->frag_off |= htons(IP6_MF);
846 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr));
849 state->offset += len;
853 EXPORT_SYMBOL(ip6_frag_next);
855 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
856 int (*output)(struct net *, struct sock *, struct sk_buff *))
858 struct sk_buff *frag;
859 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
860 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ?
861 inet6_sk(skb->sk) : NULL;
862 bool mono_delivery_time = skb->mono_delivery_time;
863 struct ip6_frag_state state;
864 unsigned int mtu, hlen, nexthdr_offset;
865 ktime_t tstamp = skb->tstamp;
868 u8 *prevhdr, nexthdr = 0;
870 err = ip6_find_1stfragopt(skb, &prevhdr);
875 nexthdr_offset = prevhdr - skb_network_header(skb);
877 mtu = ip6_skb_dst_mtu(skb);
879 /* We must not fragment if the socket is set to force MTU discovery
880 * or if the skb it not generated by a local socket.
882 if (unlikely(!skb->ignore_df && skb->len > mtu))
885 if (IP6CB(skb)->frag_max_size) {
886 if (IP6CB(skb)->frag_max_size > mtu)
889 /* don't send fragments larger than what we received */
890 mtu = IP6CB(skb)->frag_max_size;
891 if (mtu < IPV6_MIN_MTU)
896 u32 frag_size = READ_ONCE(np->frag_size);
898 if (frag_size && frag_size < mtu)
901 if (mtu < hlen + sizeof(struct frag_hdr) + 8)
903 mtu -= hlen + sizeof(struct frag_hdr);
905 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr,
906 &ipv6_hdr(skb)->saddr);
908 if (skb->ip_summed == CHECKSUM_PARTIAL &&
909 (err = skb_checksum_help(skb)))
912 prevhdr = skb_network_header(skb) + nexthdr_offset;
913 hroom = LL_RESERVED_SPACE(rt->dst.dev);
914 if (skb_has_frag_list(skb)) {
915 unsigned int first_len = skb_pagelen(skb);
916 struct ip6_fraglist_iter iter;
917 struct sk_buff *frag2;
919 if (first_len - hlen > mtu ||
920 ((first_len - hlen) & 7) ||
922 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr)))
925 skb_walk_frags(skb, frag) {
926 /* Correct geometry. */
927 if (frag->len > mtu ||
928 ((frag->len & 7) && frag->next) ||
929 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr)))
930 goto slow_path_clean;
932 /* Partially cloned skb? */
933 if (skb_shared(frag))
934 goto slow_path_clean;
939 frag->destructor = sock_wfree;
941 skb->truesize -= frag->truesize;
944 err = ip6_fraglist_init(skb, hlen, prevhdr, nexthdr, frag_id,
949 /* We prevent @rt from being freed. */
953 /* Prepare header of the next frame,
954 * before previous one went down. */
956 ip6_fraglist_prepare(skb, &iter);
958 skb_set_delivery_time(skb, tstamp, mono_delivery_time);
959 err = output(net, sk, skb);
961 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
962 IPSTATS_MIB_FRAGCREATES);
964 if (err || !iter.frag)
967 skb = ip6_fraglist_next(&iter);
973 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
974 IPSTATS_MIB_FRAGOKS);
979 kfree_skb_list(iter.frag);
981 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
982 IPSTATS_MIB_FRAGFAILS);
987 skb_walk_frags(skb, frag2) {
991 frag2->destructor = NULL;
992 skb->truesize += frag2->truesize;
998 * Fragment the datagram.
1001 ip6_frag_init(skb, hlen, mtu, rt->dst.dev->needed_tailroom,
1002 LL_RESERVED_SPACE(rt->dst.dev), prevhdr, nexthdr, frag_id,
1006 * Keep copying data until we run out.
1009 while (state.left > 0) {
1010 frag = ip6_frag_next(skb, &state);
1012 err = PTR_ERR(frag);
1017 * Put this fragment into the sending queue.
1019 skb_set_delivery_time(frag, tstamp, mono_delivery_time);
1020 err = output(net, sk, frag);
1024 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
1025 IPSTATS_MIB_FRAGCREATES);
1027 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
1028 IPSTATS_MIB_FRAGOKS);
1033 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
1037 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
1038 IPSTATS_MIB_FRAGFAILS);
1043 static inline int ip6_rt_check(const struct rt6key *rt_key,
1044 const struct in6_addr *fl_addr,
1045 const struct in6_addr *addr_cache)
1047 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) &&
1048 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache));
1051 static struct dst_entry *ip6_sk_dst_check(struct sock *sk,
1052 struct dst_entry *dst,
1053 const struct flowi6 *fl6)
1055 struct ipv6_pinfo *np = inet6_sk(sk);
1056 struct rt6_info *rt;
1061 if (dst->ops->family != AF_INET6) {
1066 rt = (struct rt6_info *)dst;
1067 /* Yes, checking route validity in not connected
1068 * case is not very simple. Take into account,
1069 * that we do not support routing by source, TOS,
1070 * and MSG_DONTROUTE --ANK (980726)
1072 * 1. ip6_rt_check(): If route was host route,
1073 * check that cached destination is current.
1074 * If it is network route, we still may
1075 * check its validity using saved pointer
1076 * to the last used address: daddr_cache.
1077 * We do not want to save whole address now,
1078 * (because main consumer of this service
1079 * is tcp, which has not this problem),
1080 * so that the last trick works only on connected
1082 * 2. oif also should be the same.
1084 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) ||
1085 #ifdef CONFIG_IPV6_SUBTREES
1086 ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) ||
1088 (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex)) {
1097 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
1098 struct dst_entry **dst, struct flowi6 *fl6)
1100 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
1101 struct neighbour *n;
1102 struct rt6_info *rt;
1107 /* The correct way to handle this would be to do
1108 * ip6_route_get_saddr, and then ip6_route_output; however,
1109 * the route-specific preferred source forces the
1110 * ip6_route_output call _before_ ip6_route_get_saddr.
1112 * In source specific routing (no src=any default route),
1113 * ip6_route_output will fail given src=any saddr, though, so
1114 * that's why we try it again later.
1116 if (ipv6_addr_any(&fl6->saddr)) {
1117 struct fib6_info *from;
1118 struct rt6_info *rt;
1120 *dst = ip6_route_output(net, sk, fl6);
1121 rt = (*dst)->error ? NULL : (struct rt6_info *)*dst;
1124 from = rt ? rcu_dereference(rt->from) : NULL;
1125 err = ip6_route_get_saddr(net, from, &fl6->daddr,
1126 sk ? READ_ONCE(inet6_sk(sk)->srcprefs) : 0,
1131 goto out_err_release;
1133 /* If we had an erroneous initial result, pretend it
1134 * never existed and let the SA-enabled version take
1137 if ((*dst)->error) {
1142 if (fl6->flowi6_oif)
1143 flags |= RT6_LOOKUP_F_IFACE;
1147 *dst = ip6_route_output_flags(net, sk, fl6, flags);
1149 err = (*dst)->error;
1151 goto out_err_release;
1153 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
1155 * Here if the dst entry we've looked up
1156 * has a neighbour entry that is in the INCOMPLETE
1157 * state and the src address from the flow is
1158 * marked as OPTIMISTIC, we release the found
1159 * dst entry and replace it instead with the
1160 * dst entry of the nexthop router
1162 rt = (struct rt6_info *) *dst;
1164 n = __ipv6_neigh_lookup_noref(rt->dst.dev,
1165 rt6_nexthop(rt, &fl6->daddr));
1166 err = n && !(READ_ONCE(n->nud_state) & NUD_VALID) ? -EINVAL : 0;
1170 struct inet6_ifaddr *ifp;
1171 struct flowi6 fl_gw6;
1174 ifp = ipv6_get_ifaddr(net, &fl6->saddr,
1177 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC);
1183 * We need to get the dst entry for the
1184 * default router instead
1187 memcpy(&fl_gw6, fl6, sizeof(struct flowi6));
1188 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr));
1189 *dst = ip6_route_output(net, sk, &fl_gw6);
1190 err = (*dst)->error;
1192 goto out_err_release;
1196 if (ipv6_addr_v4mapped(&fl6->saddr) &&
1197 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) {
1198 err = -EAFNOSUPPORT;
1199 goto out_err_release;
1208 if (err == -ENETUNREACH)
1209 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
1214 * ip6_dst_lookup - perform route lookup on flow
1215 * @net: Network namespace to perform lookup in
1216 * @sk: socket which provides route info
1217 * @dst: pointer to dst_entry * for result
1218 * @fl6: flow to lookup
1220 * This function performs a route lookup on the given flow.
1222 * It returns zero on success, or a standard errno code on error.
1224 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst,
1228 return ip6_dst_lookup_tail(net, sk, dst, fl6);
1230 EXPORT_SYMBOL_GPL(ip6_dst_lookup);
1233 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec
1234 * @net: Network namespace to perform lookup in
1235 * @sk: socket which provides route info
1236 * @fl6: flow to lookup
1237 * @final_dst: final destination address for ipsec lookup
1239 * This function performs a route lookup on the given flow.
1241 * It returns a valid dst pointer on success, or a pointer encoded
1244 struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, struct flowi6 *fl6,
1245 const struct in6_addr *final_dst)
1247 struct dst_entry *dst = NULL;
1250 err = ip6_dst_lookup_tail(net, sk, &dst, fl6);
1252 return ERR_PTR(err);
1254 fl6->daddr = *final_dst;
1256 return xfrm_lookup_route(net, dst, flowi6_to_flowi(fl6), sk, 0);
1258 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow);
1261 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow
1262 * @sk: socket which provides the dst cache and route info
1263 * @fl6: flow to lookup
1264 * @final_dst: final destination address for ipsec lookup
1265 * @connected: whether @sk is connected or not
1267 * This function performs a route lookup on the given flow with the
1268 * possibility of using the cached route in the socket if it is valid.
1269 * It will take the socket dst lock when operating on the dst cache.
1270 * As a result, this function can only be used in process context.
1272 * In addition, for a connected socket, cache the dst in the socket
1273 * if the current cache is not valid.
1275 * It returns a valid dst pointer on success, or a pointer encoded
1278 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
1279 const struct in6_addr *final_dst,
1282 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie);
1284 dst = ip6_sk_dst_check(sk, dst, fl6);
1288 dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_dst);
1289 if (connected && !IS_ERR(dst))
1290 ip6_sk_dst_store_flow(sk, dst_clone(dst), fl6);
1294 EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow);
1296 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src,
1299 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1302 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src,
1305 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1308 static void ip6_append_data_mtu(unsigned int *mtu,
1310 unsigned int fragheaderlen,
1311 struct sk_buff *skb,
1312 struct rt6_info *rt,
1313 unsigned int orig_mtu)
1315 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) {
1317 /* first fragment, reserve header_len */
1318 *mtu = orig_mtu - rt->dst.header_len;
1322 * this fragment is not first, the headers
1323 * space is regarded as data space.
1327 *maxfraglen = ((*mtu - fragheaderlen) & ~7)
1328 + fragheaderlen - sizeof(struct frag_hdr);
1332 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork,
1333 struct inet6_cork *v6_cork, struct ipcm6_cookie *ipc6,
1334 struct rt6_info *rt)
1336 struct ipv6_pinfo *np = inet6_sk(sk);
1337 unsigned int mtu, frag_size;
1338 struct ipv6_txoptions *nopt, *opt = ipc6->opt;
1340 /* callers pass dst together with a reference, set it first so
1341 * ip6_cork_release() can put it down even in case of an error.
1343 cork->base.dst = &rt->dst;
1349 if (WARN_ON(v6_cork->opt))
1352 nopt = v6_cork->opt = kzalloc(sizeof(*opt), sk->sk_allocation);
1353 if (unlikely(!nopt))
1356 nopt->tot_len = sizeof(*opt);
1357 nopt->opt_flen = opt->opt_flen;
1358 nopt->opt_nflen = opt->opt_nflen;
1360 nopt->dst0opt = ip6_opt_dup(opt->dst0opt, sk->sk_allocation);
1361 if (opt->dst0opt && !nopt->dst0opt)
1364 nopt->dst1opt = ip6_opt_dup(opt->dst1opt, sk->sk_allocation);
1365 if (opt->dst1opt && !nopt->dst1opt)
1368 nopt->hopopt = ip6_opt_dup(opt->hopopt, sk->sk_allocation);
1369 if (opt->hopopt && !nopt->hopopt)
1372 nopt->srcrt = ip6_rthdr_dup(opt->srcrt, sk->sk_allocation);
1373 if (opt->srcrt && !nopt->srcrt)
1376 /* need source address above miyazawa*/
1378 v6_cork->hop_limit = ipc6->hlimit;
1379 v6_cork->tclass = ipc6->tclass;
1380 if (rt->dst.flags & DST_XFRM_TUNNEL)
1381 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ?
1382 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(&rt->dst);
1384 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ?
1385 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(xfrm_dst_path(&rt->dst));
1387 frag_size = READ_ONCE(np->frag_size);
1388 if (frag_size && frag_size < mtu)
1391 cork->base.fragsize = mtu;
1392 cork->base.gso_size = ipc6->gso_size;
1393 cork->base.tx_flags = 0;
1394 cork->base.mark = ipc6->sockc.mark;
1395 sock_tx_timestamp(sk, ipc6->sockc.tsflags, &cork->base.tx_flags);
1397 cork->base.length = 0;
1398 cork->base.transmit_time = ipc6->sockc.transmit_time;
1403 static int __ip6_append_data(struct sock *sk,
1404 struct sk_buff_head *queue,
1405 struct inet_cork_full *cork_full,
1406 struct inet6_cork *v6_cork,
1407 struct page_frag *pfrag,
1408 int getfrag(void *from, char *to, int offset,
1409 int len, int odd, struct sk_buff *skb),
1410 void *from, size_t length, int transhdrlen,
1411 unsigned int flags, struct ipcm6_cookie *ipc6)
1413 struct sk_buff *skb, *skb_prev = NULL;
1414 struct inet_cork *cork = &cork_full->base;
1415 struct flowi6 *fl6 = &cork_full->fl.u.ip6;
1416 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu;
1417 struct ubuf_info *uarg = NULL;
1419 int dst_exthdrlen = 0;
1426 struct rt6_info *rt = (struct rt6_info *)cork->dst;
1427 bool paged, hold_tskey, extra_uref = false;
1428 struct ipv6_txoptions *opt = v6_cork->opt;
1429 int csummode = CHECKSUM_NONE;
1430 unsigned int maxnonfragsize, headersize;
1431 unsigned int wmem_alloc_delta = 0;
1433 skb = skb_peek_tail(queue);
1435 exthdrlen = opt ? opt->opt_flen : 0;
1436 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
1439 paged = !!cork->gso_size;
1440 mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize;
1443 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
1445 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
1446 (opt ? opt->opt_nflen : 0);
1448 headersize = sizeof(struct ipv6hdr) +
1449 (opt ? opt->opt_flen + opt->opt_nflen : 0) +
1450 rt->rt6i_nfheader_len;
1452 if (mtu <= fragheaderlen ||
1453 ((mtu - fragheaderlen) & ~7) + fragheaderlen <= sizeof(struct frag_hdr))
1456 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen -
1457 sizeof(struct frag_hdr);
1459 /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit
1460 * the first fragment
1462 if (headersize + transhdrlen > mtu)
1465 if (cork->length + length > mtu - headersize && ipc6->dontfrag &&
1466 (sk->sk_protocol == IPPROTO_UDP ||
1467 sk->sk_protocol == IPPROTO_ICMPV6 ||
1468 sk->sk_protocol == IPPROTO_RAW)) {
1469 ipv6_local_rxpmtu(sk, fl6, mtu - headersize +
1470 sizeof(struct ipv6hdr));
1474 if (ip6_sk_ignore_df(sk))
1475 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN;
1477 maxnonfragsize = mtu;
1479 if (cork->length + length > maxnonfragsize - headersize) {
1481 pmtu = max_t(int, mtu - headersize + sizeof(struct ipv6hdr), 0);
1482 ipv6_local_error(sk, EMSGSIZE, fl6, pmtu);
1486 /* CHECKSUM_PARTIAL only with no extension headers and when
1487 * we are not going to fragment
1489 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP &&
1490 headersize == sizeof(struct ipv6hdr) &&
1491 length <= mtu - headersize &&
1492 (!(flags & MSG_MORE) || cork->gso_size) &&
1493 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
1494 csummode = CHECKSUM_PARTIAL;
1496 if ((flags & MSG_ZEROCOPY) && length) {
1497 struct msghdr *msg = from;
1499 if (getfrag == ip_generic_getfrag && msg->msg_ubuf) {
1500 if (skb_zcopy(skb) && msg->msg_ubuf != skb_zcopy(skb))
1503 /* Leave uarg NULL if can't zerocopy, callers should
1504 * be able to handle it.
1506 if ((rt->dst.dev->features & NETIF_F_SG) &&
1507 csummode == CHECKSUM_PARTIAL) {
1510 uarg = msg->msg_ubuf;
1512 } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
1513 uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb));
1516 extra_uref = !skb_zcopy(skb); /* only ref on new uarg */
1517 if (rt->dst.dev->features & NETIF_F_SG &&
1518 csummode == CHECKSUM_PARTIAL) {
1522 uarg_to_msgzc(uarg)->zerocopy = 0;
1523 skb_zcopy_set(skb, uarg, &extra_uref);
1526 } else if ((flags & MSG_SPLICE_PAGES) && length) {
1527 if (inet_test_bit(HDRINCL, sk))
1529 if (rt->dst.dev->features & NETIF_F_SG &&
1530 getfrag == ip_generic_getfrag)
1531 /* We need an empty buffer to attach stuff to */
1534 flags &= ~MSG_SPLICE_PAGES;
1537 hold_tskey = cork->tx_flags & SKBTX_ANY_TSTAMP &&
1538 READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID;
1540 tskey = atomic_inc_return(&sk->sk_tskey) - 1;
1543 * Let's try using as much space as possible.
1544 * Use MTU if total length of the message fits into the MTU.
1545 * Otherwise, we need to reserve fragment header and
1546 * fragment alignment (= 8-15 octects, in total).
1548 * Note that we may need to "move" the data from the tail
1549 * of the buffer to the new fragment when we split
1552 * FIXME: It may be fragmented into multiple chunks
1553 * at once if non-fragmentable extension headers
1558 cork->length += length;
1562 while (length > 0) {
1563 /* Check if the remaining data fits into current packet. */
1564 copy = (cork->length <= mtu ? mtu : maxfraglen) - skb->len;
1566 copy = maxfraglen - skb->len;
1570 unsigned int datalen;
1571 unsigned int fraglen;
1572 unsigned int fraggap;
1573 unsigned int alloclen, alloc_extra;
1574 unsigned int pagedlen;
1576 /* There's no room in the current skb */
1578 fraggap = skb->len - maxfraglen;
1581 /* update mtu and maxfraglen if necessary */
1582 if (!skb || !skb_prev)
1583 ip6_append_data_mtu(&mtu, &maxfraglen,
1584 fragheaderlen, skb, rt,
1590 * If remaining data exceeds the mtu,
1591 * we know we need more fragment(s).
1593 datalen = length + fraggap;
1595 if (datalen > (cork->length <= mtu ? mtu : maxfraglen) - fragheaderlen)
1596 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
1597 fraglen = datalen + fragheaderlen;
1600 alloc_extra = hh_len;
1601 alloc_extra += dst_exthdrlen;
1602 alloc_extra += rt->dst.trailer_len;
1604 /* We just reserve space for fragment header.
1605 * Note: this may be overallocation if the message
1606 * (without MSG_MORE) fits into the MTU.
1608 alloc_extra += sizeof(struct frag_hdr);
1610 if ((flags & MSG_MORE) &&
1611 !(rt->dst.dev->features&NETIF_F_SG))
1614 (fraglen + alloc_extra < SKB_MAX_ALLOC ||
1615 !(rt->dst.dev->features & NETIF_F_SG)))
1618 alloclen = fragheaderlen + transhdrlen;
1619 pagedlen = datalen - transhdrlen;
1621 alloclen += alloc_extra;
1623 if (datalen != length + fraggap) {
1625 * this is not the last fragment, the trailer
1626 * space is regarded as data space.
1628 datalen += rt->dst.trailer_len;
1631 fraglen = datalen + fragheaderlen;
1633 copy = datalen - transhdrlen - fraggap - pagedlen;
1634 /* [!] NOTE: copy may be negative if pagedlen>0
1635 * because then the equation may reduces to -fraggap.
1637 if (copy < 0 && !(flags & MSG_SPLICE_PAGES)) {
1642 skb = sock_alloc_send_skb(sk, alloclen,
1643 (flags & MSG_DONTWAIT), &err);
1646 if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <=
1648 skb = alloc_skb(alloclen,
1656 * Fill in the control structures
1658 skb->protocol = htons(ETH_P_IPV6);
1659 skb->ip_summed = csummode;
1661 /* reserve for fragmentation and ipsec header */
1662 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
1666 * Find where to start putting bytes
1668 data = skb_put(skb, fraglen - pagedlen);
1669 skb_set_network_header(skb, exthdrlen);
1670 data += fragheaderlen;
1671 skb->transport_header = (skb->network_header +
1674 skb->csum = skb_copy_and_csum_bits(
1675 skb_prev, maxfraglen,
1676 data + transhdrlen, fraggap);
1677 skb_prev->csum = csum_sub(skb_prev->csum,
1680 pskb_trim_unique(skb_prev, maxfraglen);
1683 getfrag(from, data + transhdrlen, offset,
1684 copy, fraggap, skb) < 0) {
1688 } else if (flags & MSG_SPLICE_PAGES) {
1693 length -= copy + transhdrlen;
1698 /* Only the initial fragment is time stamped */
1699 skb_shinfo(skb)->tx_flags = cork->tx_flags;
1701 skb_shinfo(skb)->tskey = tskey;
1703 skb_zcopy_set(skb, uarg, &extra_uref);
1705 if ((flags & MSG_CONFIRM) && !skb_prev)
1706 skb_set_dst_pending_confirm(skb, 1);
1709 * Put the packet on the pending queue
1711 if (!skb->destructor) {
1712 skb->destructor = sock_wfree;
1714 wmem_alloc_delta += skb->truesize;
1716 __skb_queue_tail(queue, skb);
1723 if (!(rt->dst.dev->features&NETIF_F_SG) &&
1724 skb_tailroom(skb) >= copy) {
1728 if (getfrag(from, skb_put(skb, copy),
1729 offset, copy, off, skb) < 0) {
1730 __skb_trim(skb, off);
1734 } else if (flags & MSG_SPLICE_PAGES) {
1735 struct msghdr *msg = from;
1738 if (WARN_ON_ONCE(copy > msg->msg_iter.count))
1741 err = skb_splice_from_iter(skb, &msg->msg_iter, copy,
1746 wmem_alloc_delta += copy;
1748 int i = skb_shinfo(skb)->nr_frags;
1751 if (!sk_page_frag_refill(sk, pfrag))
1754 skb_zcopy_downgrade_managed(skb);
1755 if (!skb_can_coalesce(skb, i, pfrag->page,
1758 if (i == MAX_SKB_FRAGS)
1761 __skb_fill_page_desc(skb, i, pfrag->page,
1763 skb_shinfo(skb)->nr_frags = ++i;
1764 get_page(pfrag->page);
1766 copy = min_t(int, copy, pfrag->size - pfrag->offset);
1768 page_address(pfrag->page) + pfrag->offset,
1769 offset, copy, skb->len, skb) < 0)
1772 pfrag->offset += copy;
1773 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1775 skb->data_len += copy;
1776 skb->truesize += copy;
1777 wmem_alloc_delta += copy;
1779 err = skb_zerocopy_iter_dgram(skb, from, copy);
1787 if (wmem_alloc_delta)
1788 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
1794 net_zcopy_put_abort(uarg, extra_uref);
1795 cork->length -= length;
1796 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
1797 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
1799 atomic_dec(&sk->sk_tskey);
1803 int ip6_append_data(struct sock *sk,
1804 int getfrag(void *from, char *to, int offset, int len,
1805 int odd, struct sk_buff *skb),
1806 void *from, size_t length, int transhdrlen,
1807 struct ipcm6_cookie *ipc6, struct flowi6 *fl6,
1808 struct rt6_info *rt, unsigned int flags)
1810 struct inet_sock *inet = inet_sk(sk);
1811 struct ipv6_pinfo *np = inet6_sk(sk);
1815 if (flags&MSG_PROBE)
1817 if (skb_queue_empty(&sk->sk_write_queue)) {
1822 err = ip6_setup_cork(sk, &inet->cork, &np->cork,
1827 inet->cork.fl.u.ip6 = *fl6;
1828 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
1829 length += exthdrlen;
1830 transhdrlen += exthdrlen;
1835 return __ip6_append_data(sk, &sk->sk_write_queue, &inet->cork,
1836 &np->cork, sk_page_frag(sk), getfrag,
1837 from, length, transhdrlen, flags, ipc6);
1839 EXPORT_SYMBOL_GPL(ip6_append_data);
1841 static void ip6_cork_steal_dst(struct sk_buff *skb, struct inet_cork_full *cork)
1843 struct dst_entry *dst = cork->base.dst;
1845 cork->base.dst = NULL;
1846 skb_dst_set(skb, dst);
1849 static void ip6_cork_release(struct inet_cork_full *cork,
1850 struct inet6_cork *v6_cork)
1853 struct ipv6_txoptions *opt = v6_cork->opt;
1855 kfree(opt->dst0opt);
1856 kfree(opt->dst1opt);
1860 v6_cork->opt = NULL;
1863 if (cork->base.dst) {
1864 dst_release(cork->base.dst);
1865 cork->base.dst = NULL;
1869 struct sk_buff *__ip6_make_skb(struct sock *sk,
1870 struct sk_buff_head *queue,
1871 struct inet_cork_full *cork,
1872 struct inet6_cork *v6_cork)
1874 struct sk_buff *skb, *tmp_skb;
1875 struct sk_buff **tail_skb;
1876 struct in6_addr *final_dst;
1877 struct net *net = sock_net(sk);
1878 struct ipv6hdr *hdr;
1879 struct ipv6_txoptions *opt = v6_cork->opt;
1880 struct rt6_info *rt = (struct rt6_info *)cork->base.dst;
1881 struct flowi6 *fl6 = &cork->fl.u.ip6;
1882 unsigned char proto = fl6->flowi6_proto;
1884 skb = __skb_dequeue(queue);
1887 tail_skb = &(skb_shinfo(skb)->frag_list);
1889 /* move skb->data to ip header from ext header */
1890 if (skb->data < skb_network_header(skb))
1891 __skb_pull(skb, skb_network_offset(skb));
1892 while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1893 __skb_pull(tmp_skb, skb_network_header_len(skb));
1894 *tail_skb = tmp_skb;
1895 tail_skb = &(tmp_skb->next);
1896 skb->len += tmp_skb->len;
1897 skb->data_len += tmp_skb->len;
1898 skb->truesize += tmp_skb->truesize;
1899 tmp_skb->destructor = NULL;
1903 /* Allow local fragmentation. */
1904 skb->ignore_df = ip6_sk_ignore_df(sk);
1905 __skb_pull(skb, skb_network_header_len(skb));
1907 final_dst = &fl6->daddr;
1908 if (opt && opt->opt_flen)
1909 ipv6_push_frag_opts(skb, opt, &proto);
1910 if (opt && opt->opt_nflen)
1911 ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst, &fl6->saddr);
1913 skb_push(skb, sizeof(struct ipv6hdr));
1914 skb_reset_network_header(skb);
1915 hdr = ipv6_hdr(skb);
1917 ip6_flow_hdr(hdr, v6_cork->tclass,
1918 ip6_make_flowlabel(net, skb, fl6->flowlabel,
1919 ip6_autoflowlabel(net, sk), fl6));
1920 hdr->hop_limit = v6_cork->hop_limit;
1921 hdr->nexthdr = proto;
1922 hdr->saddr = fl6->saddr;
1923 hdr->daddr = *final_dst;
1925 skb->priority = READ_ONCE(sk->sk_priority);
1926 skb->mark = cork->base.mark;
1927 skb->tstamp = cork->base.transmit_time;
1929 ip6_cork_steal_dst(skb, cork);
1930 IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS);
1931 if (proto == IPPROTO_ICMPV6) {
1932 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
1935 if (sk->sk_socket->type == SOCK_RAW &&
1936 !inet_test_bit(HDRINCL, sk))
1937 icmp6_type = fl6->fl6_icmp_type;
1939 icmp6_type = icmp6_hdr(skb)->icmp6_type;
1940 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_type);
1941 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS);
1944 ip6_cork_release(cork, v6_cork);
1949 int ip6_send_skb(struct sk_buff *skb)
1951 struct net *net = sock_net(skb->sk);
1952 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
1955 err = ip6_local_out(net, skb->sk, skb);
1958 err = net_xmit_errno(err);
1960 IP6_INC_STATS(net, rt->rt6i_idev,
1961 IPSTATS_MIB_OUTDISCARDS);
1967 int ip6_push_pending_frames(struct sock *sk)
1969 struct sk_buff *skb;
1971 skb = ip6_finish_skb(sk);
1975 return ip6_send_skb(skb);
1977 EXPORT_SYMBOL_GPL(ip6_push_pending_frames);
1979 static void __ip6_flush_pending_frames(struct sock *sk,
1980 struct sk_buff_head *queue,
1981 struct inet_cork_full *cork,
1982 struct inet6_cork *v6_cork)
1984 struct sk_buff *skb;
1986 while ((skb = __skb_dequeue_tail(queue)) != NULL) {
1988 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)),
1989 IPSTATS_MIB_OUTDISCARDS);
1993 ip6_cork_release(cork, v6_cork);
1996 void ip6_flush_pending_frames(struct sock *sk)
1998 __ip6_flush_pending_frames(sk, &sk->sk_write_queue,
1999 &inet_sk(sk)->cork, &inet6_sk(sk)->cork);
2001 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames);
2003 struct sk_buff *ip6_make_skb(struct sock *sk,
2004 int getfrag(void *from, char *to, int offset,
2005 int len, int odd, struct sk_buff *skb),
2006 void *from, size_t length, int transhdrlen,
2007 struct ipcm6_cookie *ipc6, struct rt6_info *rt,
2008 unsigned int flags, struct inet_cork_full *cork)
2010 struct inet6_cork v6_cork;
2011 struct sk_buff_head queue;
2012 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
2015 if (flags & MSG_PROBE) {
2016 dst_release(&rt->dst);
2020 __skb_queue_head_init(&queue);
2022 cork->base.flags = 0;
2023 cork->base.addr = 0;
2024 cork->base.opt = NULL;
2026 err = ip6_setup_cork(sk, cork, &v6_cork, ipc6, rt);
2028 ip6_cork_release(cork, &v6_cork);
2029 return ERR_PTR(err);
2031 if (ipc6->dontfrag < 0)
2032 ipc6->dontfrag = inet6_test_bit(DONTFRAG, sk);
2034 err = __ip6_append_data(sk, &queue, cork, &v6_cork,
2035 ¤t->task_frag, getfrag, from,
2036 length + exthdrlen, transhdrlen + exthdrlen,
2039 __ip6_flush_pending_frames(sk, &queue, cork, &v6_cork);
2040 return ERR_PTR(err);
2043 return __ip6_make_skb(sk, &queue, cork, &v6_cork);