ipv6/tcp_ipv6.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *      TCP over IPv6
   4  *      Linux INET6 implementation
   5  *
   6  *      Authors:
   7  *      Pedro Roque             <roque@di.fc.ul.pt>
   8  *
   9  *      Based on:
  10  *      linux/net/ipv4/tcp.c
  11  *      linux/net/ipv4/tcp_input.c
  12  *      linux/net/ipv4/tcp_output.c
  13  *
  14  *      Fixes:
  15  *      Hideaki YOSHIFUJI       :       sin6_scope_id support
  16  *      YOSHIFUJI Hideaki @USAGI and:   Support IPV6_V6ONLY socket option, which
  17  *      Alexey Kuznetsov                allow both IPv4 and IPv6 sockets to bind
  18  *                                      a single port at the same time.
  19  *      YOSHIFUJI Hideaki @USAGI:       convert /proc/net/tcp6 to seq_file.
  20  */
  21
  22 #include <linux/bottom_half.h>
  23 #include <linux/module.h>
  24 #include <linux/errno.h>
  25 #include <linux/types.h>
  26 #include <linux/socket.h>
  27 #include <linux/sockios.h>
  28 #include <linux/net.h>
  29 #include <linux/jiffies.h>
  30 #include <linux/in.h>
  31 #include <linux/in6.h>
  32 #include <linux/netdevice.h>
  33 #include <linux/init.h>
  34 #include <linux/jhash.h>
  35 #include <linux/ipsec.h>
  36 #include <linux/times.h>
  37 #include <linux/slab.h>
  38 #include <linux/uaccess.h>
  39 #include <linux/ipv6.h>
  40 #include <linux/icmpv6.h>
  41 #include <linux/random.h>
  42 #include <linux/indirect_call_wrapper.h>
  43
  44 #include <net/tcp.h>
  45 #include <net/ndisc.h>
  46 #include <net/inet6_hashtables.h>
  47 #include <net/inet6_connection_sock.h>
  48 #include <net/ipv6.h>
  49 #include <net/transp_v6.h>
  50 #include <net/addrconf.h>
  51 #include <net/ip6_route.h>
  52 #include <net/ip6_checksum.h>
  53 #include <net/inet_ecn.h>
  54 #include <net/protocol.h>
  55 #include <net/xfrm.h>
  56 #include <net/snmp.h>
  57 #include <net/dsfield.h>
  58 #include <net/timewait_sock.h>
  59 #include <net/inet_common.h>
  60 #include <net/secure_seq.h>
  61 #include <net/busy_poll.h>
  62
  63 #include <linux/proc_fs.h>
  64 #include <linux/seq_file.h>
  65
  66 #include <crypto/hash.h>
  67 #include <linux/scatterlist.h>
  68
  69 #include <trace/events/tcp.h>
  70
  71 static void     tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb);
  72 static void     tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
  73                                       struct request_sock *req);
  74
  75 INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
  76
  77 static const struct inet_connection_sock_af_ops ipv6_mapped;
  78 const struct inet_connection_sock_af_ops ipv6_specific;
  79 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
  80 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
  81 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
  82 #endif
  83
  84 /* Helper returning the inet6 address from a given tcp socket.
  85  * It can be used in TCP stack instead of inet6_sk(sk).
  86  * This avoids a dereference and allow compiler optimizations.
  87  * It is a specialized version of inet6_sk_generic().
  88  */
  89 #define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
  90                                               struct tcp6_sock, tcp)->inet6)
  91
  92 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
  93 {
  94         struct dst_entry *dst = skb_dst(skb);
  95
  96         if (dst && dst_hold_safe(dst)) {
  97                 const struct rt6_info *rt = (const struct rt6_info *)dst;
  98
  99                 rcu_assign_pointer(sk->sk_rx_dst, dst);
 100                 sk->sk_rx_dst_ifindex = skb->skb_iif;
 101                 sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
 102         }
 103 }
 104
 105 static u32 tcp_v6_init_seq(const struct sk_buff *skb)
 106 {
 107         return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32,
 108                                 ipv6_hdr(skb)->saddr.s6_addr32,
 109                                 tcp_hdr(skb)->dest,
 110                                 tcp_hdr(skb)->source);
 111 }
 112
 113 static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb)
 114 {
 115         return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32,
 116                                    ipv6_hdr(skb)->saddr.s6_addr32);
 117 }
 118
 119 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
 120                               int addr_len)
 121 {
 122         /* This check is replicated from tcp_v6_connect() and intended to
 123          * prevent BPF program called below from accessing bytes that are out
 124          * of the bound specified by user in addr_len.
 125          */
 126         if (addr_len < SIN6_LEN_RFC2133)
 127                 return -EINVAL;
 128
 129         sock_owned_by_me(sk);
 130
 131         return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
 132 }
 133
 134 static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 135                           int addr_len)
 136 {
 137         struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
 138         struct inet_connection_sock *icsk = inet_csk(sk);
 139         struct in6_addr *saddr = NULL, *final_p, final;
 140         struct inet_timewait_death_row *tcp_death_row;
 141         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 142         struct inet_sock *inet = inet_sk(sk);
 143         struct tcp_sock *tp = tcp_sk(sk);
 144         struct net *net = sock_net(sk);
 145         struct ipv6_txoptions *opt;
 146         struct dst_entry *dst;
 147         struct flowi6 fl6;
 148         int addr_type;
 149         int err;
 150
 151         if (addr_len < SIN6_LEN_RFC2133)
 152                 return -EINVAL;
 153
 154         if (usin->sin6_family != AF_INET6)
 155                 return -EAFNOSUPPORT;
 156
 157         memset(&fl6, 0, sizeof(fl6));
 158
 159         if (inet6_test_bit(SNDFLOW, sk)) {
 160                 fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
 161                 IP6_ECN_flow_init(fl6.flowlabel);
 162                 if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
 163                         struct ip6_flowlabel *flowlabel;
 164                         flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
 165                         if (IS_ERR(flowlabel))
 166                                 return -EINVAL;
 167                         fl6_sock_release(flowlabel);
 168                 }
 169         }
 170
 171         /*
 172          *      connect() to INADDR_ANY means loopback (BSD'ism).
 173          */
 174
 175         if (ipv6_addr_any(&usin->sin6_addr)) {
 176                 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
 177                         ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
 178                                                &usin->sin6_addr);
 179                 else
 180                         usin->sin6_addr = in6addr_loopback;
 181         }
 182
 183         addr_type = ipv6_addr_type(&usin->sin6_addr);
 184
 185         if (addr_type & IPV6_ADDR_MULTICAST)
 186                 return -ENETUNREACH;
 187
 188         if (addr_type&IPV6_ADDR_LINKLOCAL) {
 189                 if (addr_len >= sizeof(struct sockaddr_in6) &&
 190                     usin->sin6_scope_id) {
 191                         /* If interface is set while binding, indices
 192                          * must coincide.
 193                          */
 194                         if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
 195                                 return -EINVAL;
 196
 197                         sk->sk_bound_dev_if = usin->sin6_scope_id;
 198                 }
 199
 200                 /* Connect to link-local address requires an interface */
 201                 if (!sk->sk_bound_dev_if)
 202                         return -EINVAL;
 203         }
 204
 205         if (tp->rx_opt.ts_recent_stamp &&
 206             !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
 207                 tp->rx_opt.ts_recent = 0;
 208                 tp->rx_opt.ts_recent_stamp = 0;
 209                 WRITE_ONCE(tp->write_seq, 0);
 210         }
 211
 212         sk->sk_v6_daddr = usin->sin6_addr;
 213         np->flow_label = fl6.flowlabel;
 214
 215         /*
 216          *      TCP over IPv4
 217          */
 218
 219         if (addr_type & IPV6_ADDR_MAPPED) {
 220                 u32 exthdrlen = icsk->icsk_ext_hdr_len;
 221                 struct sockaddr_in sin;
 222
 223                 if (ipv6_only_sock(sk))
 224                         return -ENETUNREACH;
 225
 226                 sin.sin_family = AF_INET;
 227                 sin.sin_port = usin->sin6_port;
 228                 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
 229
 230                 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 231                 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
 232                 if (sk_is_mptcp(sk))
 233                         mptcpv6_handle_mapped(sk, true);
 234                 sk->sk_backlog_rcv = tcp_v4_do_rcv;
 235 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 236                 tp->af_specific = &tcp_sock_ipv6_mapped_specific;
 237 #endif
 238
 239                 err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin));
 240
 241                 if (err) {
 242                         icsk->icsk_ext_hdr_len = exthdrlen;
 243                         /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 244                         WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
 245                         if (sk_is_mptcp(sk))
 246                                 mptcpv6_handle_mapped(sk, false);
 247                         sk->sk_backlog_rcv = tcp_v6_do_rcv;
 248 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 249                         tp->af_specific = &tcp_sock_ipv6_specific;
 250 #endif
 251                         goto failure;
 252                 }
 253                 np->saddr = sk->sk_v6_rcv_saddr;
 254
 255                 return err;
 256         }
 257
 258         if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
 259                 saddr = &sk->sk_v6_rcv_saddr;
 260
 261         fl6.flowi6_proto = IPPROTO_TCP;
 262         fl6.daddr = sk->sk_v6_daddr;
 263         fl6.saddr = saddr ? *saddr : np->saddr;
 264         fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
 265         fl6.flowi6_oif = sk->sk_bound_dev_if;
 266         fl6.flowi6_mark = sk->sk_mark;
 267         fl6.fl6_dport = usin->sin6_port;
 268         fl6.fl6_sport = inet->inet_sport;
 269         fl6.flowi6_uid = sk->sk_uid;
 270
 271         opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
 272         final_p = fl6_update_dst(&fl6, opt, &final);
 273
 274         security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
 275
 276         dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p);
 277         if (IS_ERR(dst)) {
 278                 err = PTR_ERR(dst);
 279                 goto failure;
 280         }
 281
 282         tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
 283         tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
 284
 285         if (!saddr) {
 286                 saddr = &fl6.saddr;
 287
 288                 err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
 289                 if (err)
 290                         goto failure;
 291         }
 292
 293         /* set the source address */
 294         np->saddr = *saddr;
 295         inet->inet_rcv_saddr = LOOPBACK4_IPV6;
 296
 297         sk->sk_gso_type = SKB_GSO_TCPV6;
 298         ip6_dst_store(sk, dst, NULL, NULL);
 299
 300         icsk->icsk_ext_hdr_len = 0;
 301         if (opt)
 302                 icsk->icsk_ext_hdr_len = opt->opt_flen +
 303                                          opt->opt_nflen;
 304
 305         tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
 306
 307         inet->inet_dport = usin->sin6_port;
 308
 309         tcp_set_state(sk, TCP_SYN_SENT);
 310         err = inet6_hash_connect(tcp_death_row, sk);
 311         if (err)
 312                 goto late_failure;
 313
 314         sk_set_txhash(sk);
 315
 316         if (likely(!tp->repair)) {
 317                 if (!tp->write_seq)
 318                         WRITE_ONCE(tp->write_seq,
 319                                    secure_tcpv6_seq(np->saddr.s6_addr32,
 320                                                     sk->sk_v6_daddr.s6_addr32,
 321                                                     inet->inet_sport,
 322                                                     inet->inet_dport));
 323                 tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32,
 324                                                    sk->sk_v6_daddr.s6_addr32);
 325         }
 326
 327         if (tcp_fastopen_defer_connect(sk, &err))
 328                 return err;
 329         if (err)
 330                 goto late_failure;
 331
 332         err = tcp_connect(sk);
 333         if (err)
 334                 goto late_failure;
 335
 336         return 0;
 337
 338 late_failure:
 339         tcp_set_state(sk, TCP_CLOSE);
 340         inet_bhash2_reset_saddr(sk);
 341 failure:
 342         inet->inet_dport = 0;
 343         sk->sk_route_caps = 0;
 344         return err;
 345 }
 346
 347 static void tcp_v6_mtu_reduced(struct sock *sk)
 348 {
 349         struct dst_entry *dst;
 350         u32 mtu;
 351
 352         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
 353                 return;
 354
 355         mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
 356
 357         /* Drop requests trying to increase our current mss.
 358          * Check done in __ip6_rt_update_pmtu() is too late.
 359          */
 360         if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache)
 361                 return;
 362
 363         dst = inet6_csk_update_pmtu(sk, mtu);
 364         if (!dst)
 365                 return;
 366
 367         if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
 368                 tcp_sync_mss(sk, dst_mtu(dst));
 369                 tcp_simple_retransmit(sk);
 370         }
 371 }
 372
 373 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 374                 u8 type, u8 code, int offset, __be32 info)
 375 {
 376         const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
 377         const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
 378         struct net *net = dev_net(skb->dev);
 379         struct request_sock *fastopen;
 380         struct ipv6_pinfo *np;
 381         struct tcp_sock *tp;
 382         __u32 seq, snd_una;
 383         struct sock *sk;
 384         bool fatal;
 385         int err;
 386
 387         sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 388                                         &hdr->daddr, th->dest,
 389                                         &hdr->saddr, ntohs(th->source),
 390                                         skb->dev->ifindex, inet6_sdif(skb));
 391
 392         if (!sk) {
 393                 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
 394                                   ICMP6_MIB_INERRORS);
 395                 return -ENOENT;
 396         }
 397
 398         if (sk->sk_state == TCP_TIME_WAIT) {
 399                 /* To increase the counter of ignored icmps for TCP-AO */
 400                 tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
 401                 inet_twsk_put(inet_twsk(sk));
 402                 return 0;
 403         }
 404         seq = ntohl(th->seq);
 405         fatal = icmpv6_err_convert(type, code, &err);
 406         if (sk->sk_state == TCP_NEW_SYN_RECV) {
 407                 tcp_req_err(sk, seq, fatal);
 408                 return 0;
 409         }
 410
 411         if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
 412                 sock_put(sk);
 413                 return 0;
 414         }
 415
 416         bh_lock_sock(sk);
 417         if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
 418                 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
 419
 420         if (sk->sk_state == TCP_CLOSE)
 421                 goto out;
 422
 423         if (static_branch_unlikely(&ip6_min_hopcount)) {
 424                 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
 425                 if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
 426                         __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
 427                         goto out;
 428                 }
 429         }
 430
 431         tp = tcp_sk(sk);
 432         /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
 433         fastopen = rcu_dereference(tp->fastopen_rsk);
 434         snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
 435         if (sk->sk_state != TCP_LISTEN &&
 436             !between(seq, snd_una, tp->snd_nxt)) {
 437                 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 438                 goto out;
 439         }
 440
 441         np = tcp_inet6_sk(sk);
 442
 443         if (type == NDISC_REDIRECT) {
 444                 if (!sock_owned_by_user(sk)) {
 445                         struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);
 446
 447                         if (dst)
 448                                 dst->ops->redirect(dst, sk, skb);
 449                 }
 450                 goto out;
 451         }
 452
 453         if (type == ICMPV6_PKT_TOOBIG) {
 454                 u32 mtu = ntohl(info);
 455
 456                 /* We are not interested in TCP_LISTEN and open_requests
 457                  * (SYN-ACKs send out by Linux are always <576bytes so
 458                  * they should go through unfragmented).
 459                  */
 460                 if (sk->sk_state == TCP_LISTEN)
 461                         goto out;
 462
 463                 if (!ip6_sk_accept_pmtu(sk))
 464                         goto out;
 465
 466                 if (mtu < IPV6_MIN_MTU)
 467                         goto out;
 468
 469                 WRITE_ONCE(tp->mtu_info, mtu);
 470
 471                 if (!sock_owned_by_user(sk))
 472                         tcp_v6_mtu_reduced(sk);
 473                 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
 474                                            &sk->sk_tsq_flags))
 475                         sock_hold(sk);
 476                 goto out;
 477         }
 478
 479
 480         /* Might be for an request_sock */
 481         switch (sk->sk_state) {
 482         case TCP_SYN_SENT:
 483         case TCP_SYN_RECV:
 484                 /* Only in fast or simultaneous open. If a fast open socket is
 485                  * already accepted it is treated as a connected one below.
 486                  */
 487                 if (fastopen && !fastopen->sk)
 488                         break;
 489
 490                 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
 491
 492                 if (!sock_owned_by_user(sk)) {
 493                         WRITE_ONCE(sk->sk_err, err);
 494                         sk_error_report(sk);            /* Wake people up to see the error (see connect in sock.c) */
 495
 496                         tcp_done(sk);
 497                 } else {
 498                         WRITE_ONCE(sk->sk_err_soft, err);
 499                 }
 500                 goto out;
 501         case TCP_LISTEN:
 502                 break;
 503         default:
 504                 /* check if this ICMP message allows revert of backoff.
 505                  * (see RFC 6069)
 506                  */
 507                 if (!fastopen && type == ICMPV6_DEST_UNREACH &&
 508                     code == ICMPV6_NOROUTE)
 509                         tcp_ld_RTO_revert(sk, seq);
 510         }
 511
 512         if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
 513                 WRITE_ONCE(sk->sk_err, err);
 514                 sk_error_report(sk);
 515         } else {
 516                 WRITE_ONCE(sk->sk_err_soft, err);
 517         }
 518 out:
 519         bh_unlock_sock(sk);
 520         sock_put(sk);
 521         return 0;
 522 }
 523
 524
 525 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
 526                               struct flowi *fl,
 527                               struct request_sock *req,
 528                               struct tcp_fastopen_cookie *foc,
 529                               enum tcp_synack_type synack_type,
 530                               struct sk_buff *syn_skb)
 531 {
 532         struct inet_request_sock *ireq = inet_rsk(req);
 533         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 534         struct ipv6_txoptions *opt;
 535         struct flowi6 *fl6 = &fl->u.ip6;
 536         struct sk_buff *skb;
 537         int err = -ENOMEM;
 538         u8 tclass;
 539
 540         /* First, grab a route. */
 541         if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
 542                                                IPPROTO_TCP)) == NULL)
 543                 goto done;
 544
 545         skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
 546
 547         if (skb) {
 548                 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
 549                                     &ireq->ir_v6_rmt_addr);
 550
 551                 fl6->daddr = ireq->ir_v6_rmt_addr;
 552                 if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
 553                         fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));
 554
 555                 tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
 556                                 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
 557                                 (np->tclass & INET_ECN_MASK) :
 558                                 np->tclass;
 559
 560                 if (!INET_ECN_is_capable(tclass) &&
 561                     tcp_bpf_ca_needs_ecn((struct sock *)req))
 562                         tclass |= INET_ECN_ECT_0;
 563
 564                 rcu_read_lock();
 565                 opt = ireq->ipv6_opt;
 566                 if (!opt)
 567                         opt = rcu_dereference(np->opt);
 568                 err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark),
 569                                opt, tclass, READ_ONCE(sk->sk_priority));
 570                 rcu_read_unlock();
 571                 err = net_xmit_eval(err);
 572         }
 573
 574 done:
 575         return err;
 576 }
 577
 578
 579 static void tcp_v6_reqsk_destructor(struct request_sock *req)
 580 {
 581         kfree(inet_rsk(req)->ipv6_opt);
 582         consume_skb(inet_rsk(req)->pktopts);
 583 }
 584
 585 #ifdef CONFIG_TCP_MD5SIG
 586 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
 587                                                    const struct in6_addr *addr,
 588                                                    int l3index)
 589 {
 590         return tcp_md5_do_lookup(sk, l3index,
 591                                  (union tcp_md5_addr *)addr, AF_INET6);
 592 }
 593
 594 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
 595                                                 const struct sock *addr_sk)
 596 {
 597         int l3index;
 598
 599         l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 600                                                  addr_sk->sk_bound_dev_if);
 601         return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
 602                                     l3index);
 603 }
 604
 605 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
 606                                  sockptr_t optval, int optlen)
 607 {
 608         struct tcp_md5sig cmd;
 609         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
 610         union tcp_ao_addr *addr;
 611         int l3index = 0;
 612         u8 prefixlen;
 613         bool l3flag;
 614         u8 flags;
 615
 616         if (optlen < sizeof(cmd))
 617                 return -EINVAL;
 618
 619         if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
 620                 return -EFAULT;
 621
 622         if (sin6->sin6_family != AF_INET6)
 623                 return -EINVAL;
 624
 625         flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 626         l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 627
 628         if (optname == TCP_MD5SIG_EXT &&
 629             cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
 630                 prefixlen = cmd.tcpm_prefixlen;
 631                 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
 632                                         prefixlen > 32))
 633                         return -EINVAL;
 634         } else {
 635                 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
 636         }
 637
 638         if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
 639             cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
 640                 struct net_device *dev;
 641
 642                 rcu_read_lock();
 643                 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
 644                 if (dev && netif_is_l3_master(dev))
 645                         l3index = dev->ifindex;
 646                 rcu_read_unlock();
 647
 648                 /* ok to reference set/not set outside of rcu;
 649                  * right now device MUST be an L3 master
 650                  */
 651                 if (!dev || !l3index)
 652                         return -EINVAL;
 653         }
 654
 655         if (!cmd.tcpm_keylen) {
 656                 if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 657                         return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 658                                               AF_INET, prefixlen,
 659                                               l3index, flags);
 660                 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 661                                       AF_INET6, prefixlen, l3index, flags);
 662         }
 663
 664         if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
 665                 return -EINVAL;
 666
 667         if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
 668                 addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];
 669
 670                 /* Don't allow keys for peers that have a matching TCP-AO key.
 671                  * See the comment in tcp_ao_add_cmd()
 672                  */
 673                 if (tcp_ao_required(sk, addr, AF_INET,
 674                                     l3flag ? l3index : -1, false))
 675                         return -EKEYREJECTED;
 676                 return tcp_md5_do_add(sk, addr,
 677                                       AF_INET, prefixlen, l3index, flags,
 678                                       cmd.tcpm_key, cmd.tcpm_keylen);
 679         }
 680
 681         addr = (union tcp_md5_addr *)&sin6->sin6_addr;
 682
 683         /* Don't allow keys for peers that have a matching TCP-AO key.
 684          * See the comment in tcp_ao_add_cmd()
 685          */
 686         if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false))
 687                 return -EKEYREJECTED;
 688
 689         return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
 690                               cmd.tcpm_key, cmd.tcpm_keylen);
 691 }
 692
 693 static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp,
 694                                    const struct in6_addr *daddr,
 695                                    const struct in6_addr *saddr,
 696                                    const struct tcphdr *th, int nbytes)
 697 {
 698         struct tcp6_pseudohdr *bp;
 699         struct scatterlist sg;
 700         struct tcphdr *_th;
 701
 702         bp = hp->scratch;
 703         /* 1. TCP pseudo-header (RFC2460) */
 704         bp->saddr = *saddr;
 705         bp->daddr = *daddr;
 706         bp->protocol = cpu_to_be32(IPPROTO_TCP);
 707         bp->len = cpu_to_be32(nbytes);
 708
 709         _th = (struct tcphdr *)(bp + 1);
 710         memcpy(_th, th, sizeof(*th));
 711         _th->check = 0;
 712
 713         sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
 714         ahash_request_set_crypt(hp->req, &sg, NULL,
 715                                 sizeof(*bp) + sizeof(*th));
 716         return crypto_ahash_update(hp->req);
 717 }
 718
 719 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
 720                                const struct in6_addr *daddr, struct in6_addr *saddr,
 721                                const struct tcphdr *th)
 722 {
 723         struct tcp_sigpool hp;
 724
 725         if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 726                 goto clear_hash_nostart;
 727
 728         if (crypto_ahash_init(hp.req))
 729                 goto clear_hash;
 730         if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2))
 731                 goto clear_hash;
 732         if (tcp_md5_hash_key(&hp, key))
 733                 goto clear_hash;
 734         ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 735         if (crypto_ahash_final(hp.req))
 736                 goto clear_hash;
 737
 738         tcp_sigpool_end(&hp);
 739         return 0;
 740
 741 clear_hash:
 742         tcp_sigpool_end(&hp);
 743 clear_hash_nostart:
 744         memset(md5_hash, 0, 16);
 745         return 1;
 746 }
 747
 748 static int tcp_v6_md5_hash_skb(char *md5_hash,
 749                                const struct tcp_md5sig_key *key,
 750                                const struct sock *sk,
 751                                const struct sk_buff *skb)
 752 {
 753         const struct tcphdr *th = tcp_hdr(skb);
 754         const struct in6_addr *saddr, *daddr;
 755         struct tcp_sigpool hp;
 756
 757         if (sk) { /* valid for establish/request sockets */
 758                 saddr = &sk->sk_v6_rcv_saddr;
 759                 daddr = &sk->sk_v6_daddr;
 760         } else {
 761                 const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 762                 saddr = &ip6h->saddr;
 763                 daddr = &ip6h->daddr;
 764         }
 765
 766         if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 767                 goto clear_hash_nostart;
 768
 769         if (crypto_ahash_init(hp.req))
 770                 goto clear_hash;
 771
 772         if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len))
 773                 goto clear_hash;
 774         if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
 775                 goto clear_hash;
 776         if (tcp_md5_hash_key(&hp, key))
 777                 goto clear_hash;
 778         ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 779         if (crypto_ahash_final(hp.req))
 780                 goto clear_hash;
 781
 782         tcp_sigpool_end(&hp);
 783         return 0;
 784
 785 clear_hash:
 786         tcp_sigpool_end(&hp);
 787 clear_hash_nostart:
 788         memset(md5_hash, 0, 16);
 789         return 1;
 790 }
 791 #endif
 792
 793 static void tcp_v6_init_req(struct request_sock *req,
 794                             const struct sock *sk_listener,
 795                             struct sk_buff *skb)
 796 {
 797         bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
 798         struct inet_request_sock *ireq = inet_rsk(req);
 799         const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
 800
 801         ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
 802         ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
 803
 804         /* So that link locals have meaning */
 805         if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
 806             ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
 807                 ireq->ir_iif = tcp_v6_iif(skb);
 808
 809         if (!TCP_SKB_CB(skb)->tcp_tw_isn &&
 810             (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
 811              np->rxopt.bits.rxinfo ||
 812              np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
 813              np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
 814                 refcount_inc(&skb->users);
 815                 ireq->pktopts = skb;
 816         }
 817 }
 818
 819 static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
 820                                           struct sk_buff *skb,
 821                                           struct flowi *fl,
 822                                           struct request_sock *req)
 823 {
 824         tcp_v6_init_req(req, sk, skb);
 825
 826         if (security_inet_conn_request(sk, skb, req))
 827                 return NULL;
 828
 829         return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP);
 830 }
 831
 832 struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
 833         .family         =       AF_INET6,
 834         .obj_size       =       sizeof(struct tcp6_request_sock),
 835         .rtx_syn_ack    =       tcp_rtx_synack,
 836         .send_ack       =       tcp_v6_reqsk_send_ack,
 837         .destructor     =       tcp_v6_reqsk_destructor,
 838         .send_reset     =       tcp_v6_send_reset,
 839         .syn_ack_timeout =      tcp_syn_ack_timeout,
 840 };
 841
 842 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
 843         .mss_clamp      =       IPV6_MIN_MTU - sizeof(struct tcphdr) -
 844                                 sizeof(struct ipv6hdr),
 845 #ifdef CONFIG_TCP_MD5SIG
 846         .req_md5_lookup =       tcp_v6_md5_lookup,
 847         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
 848 #endif
 849 #ifdef CONFIG_TCP_AO
 850         .ao_lookup      =       tcp_v6_ao_lookup_rsk,
 851         .ao_calc_key    =       tcp_v6_ao_calc_key_rsk,
 852         .ao_synack_hash =       tcp_v6_ao_synack_hash,
 853 #endif
 854 #ifdef CONFIG_SYN_COOKIES
 855         .cookie_init_seq =      cookie_v6_init_sequence,
 856 #endif
 857         .route_req      =       tcp_v6_route_req,
 858         .init_seq       =       tcp_v6_init_seq,
 859         .init_ts_off    =       tcp_v6_init_ts_off,
 860         .send_synack    =       tcp_v6_send_synack,
 861 };
 862
 863 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
 864                                  u32 ack, u32 win, u32 tsval, u32 tsecr,
 865                                  int oif, int rst, u8 tclass, __be32 label,
 866                                  u32 priority, u32 txhash, struct tcp_key *key)
 867 {
 868         const struct tcphdr *th = tcp_hdr(skb);
 869         struct tcphdr *t1;
 870         struct sk_buff *buff;
 871         struct flowi6 fl6;
 872         struct net *net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
 873         struct sock *ctl_sk = net->ipv6.tcp_sk;
 874         unsigned int tot_len = sizeof(struct tcphdr);
 875         __be32 mrst = 0, *topt;
 876         struct dst_entry *dst;
 877         __u32 mark = 0;
 878
 879         if (tsecr)
 880                 tot_len += TCPOLEN_TSTAMP_ALIGNED;
 881         if (tcp_key_is_md5(key))
 882                 tot_len += TCPOLEN_MD5SIG_ALIGNED;
 883         if (tcp_key_is_ao(key))
 884                 tot_len += tcp_ao_len_aligned(key->ao_key);
 885
 886 #ifdef CONFIG_MPTCP
 887         if (rst && !tcp_key_is_md5(key)) {
 888                 mrst = mptcp_reset_option(skb);
 889
 890                 if (mrst)
 891                         tot_len += sizeof(__be32);
 892         }
 893 #endif
 894
 895         buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
 896         if (!buff)
 897                 return;
 898
 899         skb_reserve(buff, MAX_TCP_HEADER);
 900
 901         t1 = skb_push(buff, tot_len);
 902         skb_reset_transport_header(buff);
 903
 904         /* Swap the send and the receive. */
 905         memset(t1, 0, sizeof(*t1));
 906         t1->dest = th->source;
 907         t1->source = th->dest;
 908         t1->doff = tot_len / 4;
 909         t1->seq = htonl(seq);
 910         t1->ack_seq = htonl(ack);
 911         t1->ack = !rst || !th->ack;
 912         t1->rst = rst;
 913         t1->window = htons(win);
 914
 915         topt = (__be32 *)(t1 + 1);
 916
 917         if (tsecr) {
 918                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 919                                 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
 920                 *topt++ = htonl(tsval);
 921                 *topt++ = htonl(tsecr);
 922         }
 923
 924         if (mrst)
 925                 *topt++ = mrst;
 926
 927 #ifdef CONFIG_TCP_MD5SIG
 928         if (tcp_key_is_md5(key)) {
 929                 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 930                                 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
 931                 tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key,
 932                                     &ipv6_hdr(skb)->saddr,
 933                                     &ipv6_hdr(skb)->daddr, t1);
 934         }
 935 #endif
 936 #ifdef CONFIG_TCP_AO
 937         if (tcp_key_is_ao(key)) {
 938                 *topt++ = htonl((TCPOPT_AO << 24) |
 939                                 (tcp_ao_len(key->ao_key) << 16) |
 940                                 (key->ao_key->sndid << 8) |
 941                                 (key->rcv_next));
 942
 943                 tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key,
 944                                 key->traffic_key,
 945                                 (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
 946                                 (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
 947                                 t1, key->sne);
 948         }
 949 #endif
 950
 951         memset(&fl6, 0, sizeof(fl6));
 952         fl6.daddr = ipv6_hdr(skb)->saddr;
 953         fl6.saddr = ipv6_hdr(skb)->daddr;
 954         fl6.flowlabel = label;
 955
 956         buff->ip_summed = CHECKSUM_PARTIAL;
 957
 958         __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);
 959
 960         fl6.flowi6_proto = IPPROTO_TCP;
 961         if (rt6_need_strict(&fl6.daddr) && !oif)
 962                 fl6.flowi6_oif = tcp_v6_iif(skb);
 963         else {
 964                 if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
 965                         oif = skb->skb_iif;
 966
 967                 fl6.flowi6_oif = oif;
 968         }
 969
 970         if (sk) {
 971                 if (sk->sk_state == TCP_TIME_WAIT)
 972                         mark = inet_twsk(sk)->tw_mark;
 973                 else
 974                         mark = READ_ONCE(sk->sk_mark);
 975                 skb_set_delivery_time(buff, tcp_transmit_time(sk), true);
 976         }
 977         if (txhash) {
 978                 /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
 979                 skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4);
 980         }
 981         fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
 982         fl6.fl6_dport = t1->dest;
 983         fl6.fl6_sport = t1->source;
 984         fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
 985         security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6));
 986
 987         /* Pass a socket to ip6_dst_lookup either it is for RST
 988          * Underlying function will use this to retrieve the network
 989          * namespace
 990          */
 991         if (sk && sk->sk_state != TCP_TIME_WAIT)
 992                 dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/
 993         else
 994                 dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL);
 995         if (!IS_ERR(dst)) {
 996                 skb_dst_set(buff, dst);
 997                 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL,
 998                          tclass & ~INET_ECN_MASK, priority);
 999                 TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
1000                 if (rst)
1001                         TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
1002                 return;
1003         }
1004
1005         kfree_skb(buff);
1006 }
1007
1008 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
1009 {
1010         const struct tcphdr *th = tcp_hdr(skb);
1011         struct ipv6hdr *ipv6h = ipv6_hdr(skb);
1012         const __u8 *md5_hash_location = NULL;
1013 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1014         bool allocated_traffic_key = false;
1015 #endif
1016         const struct tcp_ao_hdr *aoh;
1017         struct tcp_key key = {};
1018         u32 seq = 0, ack_seq = 0;
1019         __be32 label = 0;
1020         u32 priority = 0;
1021         struct net *net;
1022         u32 txhash = 0;
1023         int oif = 0;
1024 #ifdef CONFIG_TCP_MD5SIG
1025         unsigned char newhash[16];
1026         int genhash;
1027         struct sock *sk1 = NULL;
1028 #endif
1029
1030         if (th->rst)
1031                 return;
1032
1033         /* If sk not NULL, it means we did a successful lookup and incoming
1034          * route had to be correct. prequeue might have dropped our dst.
1035          */
1036         if (!sk && !ipv6_unicast_destination(skb))
1037                 return;
1038
1039         net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
1040         /* Invalid TCP option size or twice included auth */
1041         if (tcp_parse_auth_options(th, &md5_hash_location, &aoh))
1042                 return;
1043 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1044         rcu_read_lock();
1045 #endif
1046 #ifdef CONFIG_TCP_MD5SIG
1047         if (sk && sk_fullsock(sk)) {
1048                 int l3index;
1049
1050                 /* sdif set, means packet ingressed via a device
1051                  * in an L3 domain and inet_iif is set to it.
1052                  */
1053                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1054                 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
1055                 if (key.md5_key)
1056                         key.type = TCP_KEY_MD5;
1057         } else if (md5_hash_location) {
1058                 int dif = tcp_v6_iif_l3_slave(skb);
1059                 int sdif = tcp_v6_sdif(skb);
1060                 int l3index;
1061
1062                 /*
1063                  * active side is lost. Try to find listening socket through
1064                  * source port, and then find md5 key through listening socket.
1065                  * we are not loose security here:
1066                  * Incoming packet is checked with md5 hash with finding key,
1067                  * no RST generated if md5 hash doesn't match.
1068                  */
1069                 sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1070                                             NULL, 0, &ipv6h->saddr, th->source,
1071                                             &ipv6h->daddr, ntohs(th->source),
1072                                             dif, sdif);
1073                 if (!sk1)
1074                         goto out;
1075
1076                 /* sdif set, means packet ingressed via a device
1077                  * in an L3 domain and dif is set to it.
1078                  */
1079                 l3index = tcp_v6_sdif(skb) ? dif : 0;
1080
1081                 key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
1082                 if (!key.md5_key)
1083                         goto out;
1084                 key.type = TCP_KEY_MD5;
1085
1086                 genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb);
1087                 if (genhash || memcmp(md5_hash_location, newhash, 16) != 0)
1088                         goto out;
1089         }
1090 #endif
1091
1092         if (th->ack)
1093                 seq = ntohl(th->ack_seq);
1094         else
1095                 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1096                           (th->doff << 2);
1097
1098 #ifdef CONFIG_TCP_AO
1099         if (aoh) {
1100                 int l3index;
1101
1102                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1103                 if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
1104                                          &key.ao_key, &key.traffic_key,
1105                                          &allocated_traffic_key,
1106                                          &key.rcv_next, &key.sne))
1107                         goto out;
1108                 key.type = TCP_KEY_AO;
1109         }
1110 #endif
1111
1112         if (sk) {
1113                 oif = sk->sk_bound_dev_if;
1114                 if (sk_fullsock(sk)) {
1115                         trace_tcp_send_reset(sk, skb);
1116                         if (inet6_test_bit(REPFLOW, sk))
1117                                 label = ip6_flowlabel(ipv6h);
1118                         priority = READ_ONCE(sk->sk_priority);
1119                         txhash = sk->sk_txhash;
1120                 }
1121                 if (sk->sk_state == TCP_TIME_WAIT) {
1122                         label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1123                         priority = inet_twsk(sk)->tw_priority;
1124                         txhash = inet_twsk(sk)->tw_txhash;
1125                 }
1126         } else {
1127                 if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
1128                         label = ip6_flowlabel(ipv6h);
1129         }
1130
1131         tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1,
1132                              ipv6_get_dsfield(ipv6h), label, priority, txhash,
1133                              &key);
1134
1135 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1136 out:
1137         if (allocated_traffic_key)
1138                 kfree(key.traffic_key);
1139         rcu_read_unlock();
1140 #endif
1141 }
1142
1143 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1144                             u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1145                             struct tcp_key *key, u8 tclass,
1146                             __be32 label, u32 priority, u32 txhash)
1147 {
1148         tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0,
1149                              tclass, label, priority, txhash, key);
1150 }
1151
1152 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
1153 {
1154         struct inet_timewait_sock *tw = inet_twsk(sk);
1155         struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1156         struct tcp_key key = {};
1157 #ifdef CONFIG_TCP_AO
1158         struct tcp_ao_info *ao_info;
1159
1160         if (static_branch_unlikely(&tcp_ao_needed.key)) {
1161
1162                 /* FIXME: the segment to-be-acked is not verified yet */
1163                 ao_info = rcu_dereference(tcptw->ao_info);
1164                 if (ao_info) {
1165                         const struct tcp_ao_hdr *aoh;
1166
1167                         /* Invalid TCP option size or twice included auth */
1168                         if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1169                                 goto out;
1170                         if (aoh)
1171                                 key.ao_key = tcp_ao_established_key(ao_info,
1172                                                 aoh->rnext_keyid, -1);
1173                 }
1174         }
1175         if (key.ao_key) {
1176                 struct tcp_ao_key *rnext_key;
1177
1178                 key.traffic_key = snd_other_key(key.ao_key);
1179                 /* rcv_next switches to our rcv_next */
1180                 rnext_key = READ_ONCE(ao_info->rnext_key);
1181                 key.rcv_next = rnext_key->rcvid;
1182                 key.sne = READ_ONCE(ao_info->snd_sne);
1183                 key.type = TCP_KEY_AO;
1184 #else
1185         if (0) {
1186 #endif
1187 #ifdef CONFIG_TCP_MD5SIG
1188         } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1189                 key.md5_key = tcp_twsk_md5_key(tcptw);
1190                 if (key.md5_key)
1191                         key.type = TCP_KEY_MD5;
1192 #endif
1193         }
1194
1195         tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
1196                         tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1197                         tcp_tw_tsval(tcptw),
1198                         tcptw->tw_ts_recent, tw->tw_bound_dev_if, &key,
1199                         tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), tw->tw_priority,
1200                         tw->tw_txhash);
1201
1202 #ifdef CONFIG_TCP_AO
1203 out:
1204 #endif
1205         inet_twsk_put(tw);
1206 }
1207
1208 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1209                                   struct request_sock *req)
1210 {
1211         struct tcp_key key = {};
1212
1213 #ifdef CONFIG_TCP_AO
1214         if (static_branch_unlikely(&tcp_ao_needed.key) &&
1215             tcp_rsk_used_ao(req)) {
1216                 const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
1217                 const struct tcp_ao_hdr *aoh;
1218                 int l3index;
1219
1220                 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1221                 /* Invalid TCP option size or twice included auth */
1222                 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1223                         return;
1224                 if (!aoh)
1225                         return;
1226                 key.ao_key = tcp_ao_do_lookup(sk, l3index,
1227                                               (union tcp_ao_addr *)addr,
1228                                               AF_INET6, aoh->rnext_keyid, -1);
1229                 if (unlikely(!key.ao_key)) {
1230                         /* Send ACK with any matching MKT for the peer */
1231                         key.ao_key = tcp_ao_do_lookup(sk, l3index,
1232                                                       (union tcp_ao_addr *)addr,
1233                                                       AF_INET6, -1, -1);
1234                         /* Matching key disappeared (user removed the key?)
1235                          * let the handshake timeout.
1236                          */
1237                         if (!key.ao_key) {
1238                                 net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
1239                                                      addr,
1240                                                      ntohs(tcp_hdr(skb)->source),
1241                                                      &ipv6_hdr(skb)->daddr,
1242                                                      ntohs(tcp_hdr(skb)->dest));
1243                                 return;
1244                         }
1245                 }
1246                 key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1247                 if (!key.traffic_key)
1248                         return;
1249
1250                 key.type = TCP_KEY_AO;
1251                 key.rcv_next = aoh->keyid;
1252                 tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
1253 #else
1254         if (0) {
1255 #endif
1256 #ifdef CONFIG_TCP_MD5SIG
1257         } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1258                 int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1259
1260                 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr,
1261                                                    l3index);
1262                 if (key.md5_key)
1263                         key.type = TCP_KEY_MD5;
1264 #endif
1265         }
1266
1267         /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1268          * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1269          */
1270         /* RFC 7323 2.3
1271          * The window field (SEG.WND) of every outgoing segment, with the
1272          * exception of <SYN> segments, MUST be right-shifted by
1273          * Rcv.Wind.Shift bits:
1274          */
1275         tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
1276                         tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1277                         tcp_rsk(req)->rcv_nxt,
1278                         req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
1279                         tcp_rsk_tsval(tcp_rsk(req)),
1280                         READ_ONCE(req->ts_recent), sk->sk_bound_dev_if,
1281                         &key, ipv6_get_dsfield(ipv6_hdr(skb)), 0,
1282                         READ_ONCE(sk->sk_priority),
1283                         READ_ONCE(tcp_rsk(req)->txhash));
1284         if (tcp_key_is_ao(&key))
1285                 kfree(key.traffic_key);
1286 }
1287
1288
1289 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1290 {
1291 #ifdef CONFIG_SYN_COOKIES
1292         const struct tcphdr *th = tcp_hdr(skb);
1293
1294         if (!th->syn)
1295                 sk = cookie_v6_check(sk, skb);
1296 #endif
1297         return sk;
1298 }
1299
1300 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1301                          struct tcphdr *th, u32 *cookie)
1302 {
1303         u16 mss = 0;
1304 #ifdef CONFIG_SYN_COOKIES
1305         mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
1306                                     &tcp_request_sock_ipv6_ops, sk, th);
1307         if (mss) {
1308                 *cookie = __cookie_v6_init_sequence(iph, th, &mss);
1309                 tcp_synq_overflow(sk);
1310         }
1311 #endif
1312         return mss;
1313 }
1314
1315 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1316 {
1317         if (skb->protocol == htons(ETH_P_IP))
1318                 return tcp_v4_conn_request(sk, skb);
1319
1320         if (!ipv6_unicast_destination(skb))
1321                 goto drop;
1322
1323         if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
1324                 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1325                 return 0;
1326         }
1327
1328         return tcp_conn_request(&tcp6_request_sock_ops,
1329                                 &tcp_request_sock_ipv6_ops, sk, skb);
1330
1331 drop:
1332         tcp_listendrop(sk);
1333         return 0; /* don't send reset */
1334 }
1335
1336 static void tcp_v6_restore_cb(struct sk_buff *skb)
1337 {
1338         /* We need to move header back to the beginning if xfrm6_policy_check()
1339          * and tcp_v6_fill_cb() are going to be called again.
1340          * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1341          */
1342         memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1343                 sizeof(struct inet6_skb_parm));
1344 }
1345
1346 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1347                                          struct request_sock *req,
1348                                          struct dst_entry *dst,
1349                                          struct request_sock *req_unhash,
1350                                          bool *own_req)
1351 {
1352         struct inet_request_sock *ireq;
1353         struct ipv6_pinfo *newnp;
1354         const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1355         struct ipv6_txoptions *opt;
1356         struct inet_sock *newinet;
1357         bool found_dup_sk = false;
1358         struct tcp_sock *newtp;
1359         struct sock *newsk;
1360 #ifdef CONFIG_TCP_MD5SIG
1361         struct tcp_md5sig_key *key;
1362         int l3index;
1363 #endif
1364         struct flowi6 fl6;
1365
1366         if (skb->protocol == htons(ETH_P_IP)) {
1367                 /*
1368                  *      v6 mapped
1369                  */
1370
1371                 newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst,
1372                                              req_unhash, own_req);
1373
1374                 if (!newsk)
1375                         return NULL;
1376
1377                 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1378
1379                 newnp = tcp_inet6_sk(newsk);
1380                 newtp = tcp_sk(newsk);
1381
1382                 memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1383
1384                 newnp->saddr = newsk->sk_v6_rcv_saddr;
1385
1386                 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1387                 if (sk_is_mptcp(newsk))
1388                         mptcpv6_handle_mapped(newsk, true);
1389                 newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1390 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1391                 newtp->af_specific = &tcp_sock_ipv6_mapped_specific;
1392 #endif
1393
1394                 newnp->ipv6_mc_list = NULL;
1395                 newnp->ipv6_ac_list = NULL;
1396                 newnp->ipv6_fl_list = NULL;
1397                 newnp->pktoptions  = NULL;
1398                 newnp->opt         = NULL;
1399                 newnp->mcast_oif   = inet_iif(skb);
1400                 newnp->mcast_hops  = ip_hdr(skb)->ttl;
1401                 newnp->rcv_flowinfo = 0;
1402                 if (inet6_test_bit(REPFLOW, sk))
1403                         newnp->flow_label = 0;
1404
1405                 /*
1406                  * No need to charge this sock to the relevant IPv6 refcnt debug socks count
1407                  * here, tcp_create_openreq_child now does this for us, see the comment in
1408                  * that function for the gory details. -acme
1409                  */
1410
1411                 /* It is tricky place. Until this moment IPv4 tcp
1412                    worked with IPv6 icsk.icsk_af_ops.
1413                    Sync it now.
1414                  */
1415                 tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie);
1416
1417                 return newsk;
1418         }
1419
1420         ireq = inet_rsk(req);
1421
1422         if (sk_acceptq_is_full(sk))
1423                 goto out_overflow;
1424
1425         if (!dst) {
1426                 dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP);
1427                 if (!dst)
1428                         goto out;
1429         }
1430
1431         newsk = tcp_create_openreq_child(sk, req, skb);
1432         if (!newsk)
1433                 goto out_nonewsk;
1434
1435         /*
1436          * No need to charge this sock to the relevant IPv6 refcnt debug socks
1437          * count here, tcp_create_openreq_child now does this for us, see the
1438          * comment in that function for the gory details. -acme
1439          */
1440
1441         newsk->sk_gso_type = SKB_GSO_TCPV6;
1442         ip6_dst_store(newsk, dst, NULL, NULL);
1443         inet6_sk_rx_dst_set(newsk, skb);
1444
1445         inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1446
1447         newtp = tcp_sk(newsk);
1448         newinet = inet_sk(newsk);
1449         newnp = tcp_inet6_sk(newsk);
1450
1451         memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1452
1453         newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr;
1454         newnp->saddr = ireq->ir_v6_loc_addr;
1455         newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr;
1456         newsk->sk_bound_dev_if = ireq->ir_iif;
1457
1458         /* Now IPv6 options...
1459
1460            First: no IPv4 options.
1461          */
1462         newinet->inet_opt = NULL;
1463         newnp->ipv6_mc_list = NULL;
1464         newnp->ipv6_ac_list = NULL;
1465         newnp->ipv6_fl_list = NULL;
1466
1467         /* Clone RX bits */
1468         newnp->rxopt.all = np->rxopt.all;
1469
1470         newnp->pktoptions = NULL;
1471         newnp->opt        = NULL;
1472         newnp->mcast_oif  = tcp_v6_iif(skb);
1473         newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1474         newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
1475         if (inet6_test_bit(REPFLOW, sk))
1476                 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));
1477
1478         /* Set ToS of the new socket based upon the value of incoming SYN.
1479          * ECT bits are set later in tcp_init_transfer().
1480          */
1481         if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1482                 newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1483
1484         /* Clone native IPv6 options from listening socket (if any)
1485
1486            Yes, keeping reference count would be much more clever,
1487            but we make one more one thing there: reattach optmem
1488            to newsk.
1489          */
1490         opt = ireq->ipv6_opt;
1491         if (!opt)
1492                 opt = rcu_dereference(np->opt);
1493         if (opt) {
1494                 opt = ipv6_dup_options(newsk, opt);
1495                 RCU_INIT_POINTER(newnp->opt, opt);
1496         }
1497         inet_csk(newsk)->icsk_ext_hdr_len = 0;
1498         if (opt)
1499                 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1500                                                     opt->opt_flen;
1501
1502         tcp_ca_openreq_child(newsk, dst);
1503
1504         tcp_sync_mss(newsk, dst_mtu(dst));
1505         newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1506
1507         tcp_initialize_rcv_mss(newsk);
1508
1509         newinet->inet_daddr = newinet->inet_saddr = LOOPBACK4_IPV6;
1510         newinet->inet_rcv_saddr = LOOPBACK4_IPV6;
1511
1512 #ifdef CONFIG_TCP_MD5SIG
1513         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1514
1515         if (!tcp_rsk_used_ao(req)) {
1516                 /* Copy over the MD5 key from the original socket */
1517                 key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
1518                 if (key) {
1519                         const union tcp_md5_addr *addr;
1520
1521                         addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
1522                         if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) {
1523                                 inet_csk_prepare_forced_close(newsk);
1524                                 tcp_done(newsk);
1525                                 goto out;
1526                         }
1527                 }
1528         }
1529 #endif
1530 #ifdef CONFIG_TCP_AO
1531         /* Copy over tcp_ao_info if any */
1532         if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
1533                 goto out; /* OOM */
1534 #endif
1535
1536         if (__inet_inherit_port(sk, newsk) < 0) {
1537                 inet_csk_prepare_forced_close(newsk);
1538                 tcp_done(newsk);
1539                 goto out;
1540         }
1541         *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1542                                        &found_dup_sk);
1543         if (*own_req) {
1544                 tcp_move_syn(newtp, req);
1545
1546                 /* Clone pktoptions received with SYN, if we own the req */
1547                 if (ireq->pktopts) {
1548                         newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk);
1549                         consume_skb(ireq->pktopts);
1550                         ireq->pktopts = NULL;
1551                         if (newnp->pktoptions)
1552                                 tcp_v6_restore_cb(newnp->pktoptions);
1553                 }
1554         } else {
1555                 if (!req_unhash && found_dup_sk) {
1556                         /* This code path should only be executed in the
1557                          * syncookie case only
1558                          */
1559                         bh_unlock_sock(newsk);
1560                         sock_put(newsk);
1561                         newsk = NULL;
1562                 }
1563         }
1564
1565         return newsk;
1566
1567 out_overflow:
1568         __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1569 out_nonewsk:
1570         dst_release(dst);
1571 out:
1572         tcp_listendrop(sk);
1573         return NULL;
1574 }
1575
1576 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1577                                                            u32));
1578 /* The socket must have it's spinlock held when we get
1579  * here, unless it is a TCP_LISTEN socket.
1580  *
1581  * We have a potential double-lock case here, so even when
1582  * doing backlog processing we use the BH locking scheme.
1583  * This is because we cannot sleep with the original spinlock
1584  * held.
1585  */
1586 INDIRECT_CALLABLE_SCOPE
1587 int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1588 {
1589         struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1590         struct sk_buff *opt_skb = NULL;
1591         enum skb_drop_reason reason;
1592         struct tcp_sock *tp;
1593
1594         /* Imagine: socket is IPv6. IPv4 packet arrives,
1595            goes to IPv4 receive handler and backlogged.
1596            From backlog it always goes here. Kerboom...
1597            Fortunately, tcp_rcv_established and rcv_established
1598            handle them correctly, but it is not case with
1599            tcp_v6_hnd_req and tcp_v6_send_reset().   --ANK
1600          */
1601
1602         if (skb->protocol == htons(ETH_P_IP))
1603                 return tcp_v4_do_rcv(sk, skb);
1604
1605         /*
1606          *      socket locking is here for SMP purposes as backlog rcv
1607          *      is currently called with bh processing disabled.
1608          */
1609
1610         /* Do Stevens' IPV6_PKTOPTIONS.
1611
1612            Yes, guys, it is the only place in our code, where we
1613            may make it not affecting IPv4.
1614            The rest of code is protocol independent,
1615            and I do not like idea to uglify IPv4.
1616
1617            Actually, all the idea behind IPV6_PKTOPTIONS
1618            looks not very well thought. For now we latch
1619            options, received in the last packet, enqueued
1620            by tcp. Feel free to propose better solution.
1621                                                --ANK (980728)
1622          */
1623         if (np->rxopt.all)
1624                 opt_skb = skb_clone_and_charge_r(skb, sk);
1625
1626         reason = SKB_DROP_REASON_NOT_SPECIFIED;
1627         if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1628                 struct dst_entry *dst;
1629
1630                 dst = rcu_dereference_protected(sk->sk_rx_dst,
1631                                                 lockdep_sock_is_held(sk));
1632
1633                 sock_rps_save_rxhash(sk, skb);
1634                 sk_mark_napi_id(sk, skb);
1635                 if (dst) {
1636                         if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1637                             INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1638                                             dst, sk->sk_rx_dst_cookie) == NULL) {
1639                                 RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1640                                 dst_release(dst);
1641                         }
1642                 }
1643
1644                 tcp_rcv_established(sk, skb);
1645                 if (opt_skb)
1646                         goto ipv6_pktoptions;
1647                 return 0;
1648         }
1649
1650         if (tcp_checksum_complete(skb))
1651                 goto csum_err;
1652
1653         if (sk->sk_state == TCP_LISTEN) {
1654                 struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1655
1656                 if (!nsk)
1657                         goto discard;
1658
1659                 if (nsk != sk) {
1660                         if (tcp_child_process(sk, nsk, skb))
1661                                 goto reset;
1662                         if (opt_skb)
1663                                 __kfree_skb(opt_skb);
1664                         return 0;
1665                 }
1666         } else
1667                 sock_rps_save_rxhash(sk, skb);
1668
1669         if (tcp_rcv_state_process(sk, skb))
1670                 goto reset;
1671         if (opt_skb)
1672                 goto ipv6_pktoptions;
1673         return 0;
1674
1675 reset:
1676         tcp_v6_send_reset(sk, skb);
1677 discard:
1678         if (opt_skb)
1679                 __kfree_skb(opt_skb);
1680         kfree_skb_reason(skb, reason);
1681         return 0;
1682 csum_err:
1683         reason = SKB_DROP_REASON_TCP_CSUM;
1684         trace_tcp_bad_csum(skb);
1685         TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1686         TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1687         goto discard;
1688
1689
1690 ipv6_pktoptions:
1691         /* Do you ask, what is it?
1692
1693            1. skb was enqueued by tcp.
1694            2. skb is added to tail of read queue, rather than out of order.
1695            3. socket is not in passive state.
1696            4. Finally, it really contains options, which user wants to receive.
1697          */
1698         tp = tcp_sk(sk);
1699         if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1700             !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1701                 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1702                         WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb));
1703                 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1704                         WRITE_ONCE(np->mcast_hops,
1705                                    ipv6_hdr(opt_skb)->hop_limit);
1706                 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1707                         np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
1708                 if (inet6_test_bit(REPFLOW, sk))
1709                         np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
1710                 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
1711                         tcp_v6_restore_cb(opt_skb);
1712                         opt_skb = xchg(&np->pktoptions, opt_skb);
1713                 } else {
1714                         __kfree_skb(opt_skb);
1715                         opt_skb = xchg(&np->pktoptions, NULL);
1716                 }
1717         }
1718
1719         consume_skb(opt_skb);
1720         return 0;
1721 }
1722
1723 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1724                            const struct tcphdr *th)
1725 {
1726         /* This is tricky: we move IP6CB at its correct location into
1727          * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1728          * _decode_session6() uses IP6CB().
1729          * barrier() makes sure compiler won't play aliasing games.
1730          */
1731         memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1732                 sizeof(struct inet6_skb_parm));
1733         barrier();
1734
1735         TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1736         TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1737                                     skb->len - th->doff*4);
1738         TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1739         TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1740         TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1741         TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
1742         TCP_SKB_CB(skb)->sacked = 0;
1743         TCP_SKB_CB(skb)->has_rxtstamp =
1744                         skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1745 }
1746
1747 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1748 {
1749         enum skb_drop_reason drop_reason;
1750         int sdif = inet6_sdif(skb);
1751         int dif = inet6_iif(skb);
1752         const struct tcphdr *th;
1753         const struct ipv6hdr *hdr;
1754         bool refcounted;
1755         struct sock *sk;
1756         int ret;
1757         struct net *net = dev_net(skb->dev);
1758
1759         drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1760         if (skb->pkt_type != PACKET_HOST)
1761                 goto discard_it;
1762
1763         /*
1764          *      Count it even if it's bad.
1765          */
1766         __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1767
1768         if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1769                 goto discard_it;
1770
1771         th = (const struct tcphdr *)skb->data;
1772
1773         if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1774                 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1775                 goto bad_packet;
1776         }
1777         if (!pskb_may_pull(skb, th->doff*4))
1778                 goto discard_it;
1779
1780         if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1781                 goto csum_error;
1782
1783         th = (const struct tcphdr *)skb->data;
1784         hdr = ipv6_hdr(skb);
1785
1786 lookup:
1787         sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th),
1788                                 th->source, th->dest, inet6_iif(skb), sdif,
1789                                 &refcounted);
1790         if (!sk)
1791                 goto no_tcp_socket;
1792
1793 process:
1794         if (sk->sk_state == TCP_TIME_WAIT)
1795                 goto do_time_wait;
1796
1797         if (sk->sk_state == TCP_NEW_SYN_RECV) {
1798                 struct request_sock *req = inet_reqsk(sk);
1799                 bool req_stolen = false;
1800                 struct sock *nsk;
1801
1802                 sk = req->rsk_listener;
1803                 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
1804                         drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1805                 else
1806                         drop_reason = tcp_inbound_hash(sk, req, skb,
1807                                                        &hdr->saddr, &hdr->daddr,
1808                                                        AF_INET6, dif, sdif);
1809                 if (drop_reason) {
1810                         sk_drops_add(sk, skb);
1811                         reqsk_put(req);
1812                         goto discard_it;
1813                 }
1814                 if (tcp_checksum_complete(skb)) {
1815                         reqsk_put(req);
1816                         goto csum_error;
1817                 }
1818                 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1819                         nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
1820                         if (!nsk) {
1821                                 inet_csk_reqsk_queue_drop_and_put(sk, req);
1822                                 goto lookup;
1823                         }
1824                         sk = nsk;
1825                         /* reuseport_migrate_sock() has already held one sk_refcnt
1826                          * before returning.
1827                          */
1828                 } else {
1829                         sock_hold(sk);
1830                 }
1831                 refcounted = true;
1832                 nsk = NULL;
1833                 if (!tcp_filter(sk, skb)) {
1834                         th = (const struct tcphdr *)skb->data;
1835                         hdr = ipv6_hdr(skb);
1836                         tcp_v6_fill_cb(skb, hdr, th);
1837                         nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1838                 } else {
1839                         drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1840                 }
1841                 if (!nsk) {
1842                         reqsk_put(req);
1843                         if (req_stolen) {
1844                                 /* Another cpu got exclusive access to req
1845                                  * and created a full blown socket.
1846                                  * Try to feed this packet to this socket
1847                                  * instead of discarding it.
1848                                  */
1849                                 tcp_v6_restore_cb(skb);
1850                                 sock_put(sk);
1851                                 goto lookup;
1852                         }
1853                         goto discard_and_relse;
1854                 }
1855                 nf_reset_ct(skb);
1856                 if (nsk == sk) {
1857                         reqsk_put(req);
1858                         tcp_v6_restore_cb(skb);
1859                 } else if (tcp_child_process(sk, nsk, skb)) {
1860                         tcp_v6_send_reset(nsk, skb);
1861                         goto discard_and_relse;
1862                 } else {
1863                         sock_put(sk);
1864                         return 0;
1865                 }
1866         }
1867
1868         if (static_branch_unlikely(&ip6_min_hopcount)) {
1869                 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
1870                 if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
1871                         __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1872                         drop_reason = SKB_DROP_REASON_TCP_MINTTL;
1873                         goto discard_and_relse;
1874                 }
1875         }
1876
1877         if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
1878                 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1879                 goto discard_and_relse;
1880         }
1881
1882         drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr,
1883                                        AF_INET6, dif, sdif);
1884         if (drop_reason)
1885                 goto discard_and_relse;
1886
1887         nf_reset_ct(skb);
1888
1889         if (tcp_filter(sk, skb)) {
1890                 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1891                 goto discard_and_relse;
1892         }
1893         th = (const struct tcphdr *)skb->data;
1894         hdr = ipv6_hdr(skb);
1895         tcp_v6_fill_cb(skb, hdr, th);
1896
1897         skb->dev = NULL;
1898
1899         if (sk->sk_state == TCP_LISTEN) {
1900                 ret = tcp_v6_do_rcv(sk, skb);
1901                 goto put_and_return;
1902         }
1903
1904         sk_incoming_cpu_update(sk);
1905
1906         bh_lock_sock_nested(sk);
1907         tcp_segs_in(tcp_sk(sk), skb);
1908         ret = 0;
1909         if (!sock_owned_by_user(sk)) {
1910                 ret = tcp_v6_do_rcv(sk, skb);
1911         } else {
1912                 if (tcp_add_backlog(sk, skb, &drop_reason))
1913                         goto discard_and_relse;
1914         }
1915         bh_unlock_sock(sk);
1916 put_and_return:
1917         if (refcounted)
1918                 sock_put(sk);
1919         return ret ? -1 : 0;
1920
1921 no_tcp_socket:
1922         drop_reason = SKB_DROP_REASON_NO_SOCKET;
1923         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1924                 goto discard_it;
1925
1926         tcp_v6_fill_cb(skb, hdr, th);
1927
1928         if (tcp_checksum_complete(skb)) {
1929 csum_error:
1930                 drop_reason = SKB_DROP_REASON_TCP_CSUM;
1931                 trace_tcp_bad_csum(skb);
1932                 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1933 bad_packet:
1934                 __TCP_INC_STATS(net, TCP_MIB_INERRS);
1935         } else {
1936                 tcp_v6_send_reset(NULL, skb);
1937         }
1938
1939 discard_it:
1940         SKB_DR_OR(drop_reason, NOT_SPECIFIED);
1941         kfree_skb_reason(skb, drop_reason);
1942         return 0;
1943
1944 discard_and_relse:
1945         sk_drops_add(sk, skb);
1946         if (refcounted)
1947                 sock_put(sk);
1948         goto discard_it;
1949
1950 do_time_wait:
1951         if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1952                 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1953                 inet_twsk_put(inet_twsk(sk));
1954                 goto discard_it;
1955         }
1956
1957         tcp_v6_fill_cb(skb, hdr, th);
1958
1959         if (tcp_checksum_complete(skb)) {
1960                 inet_twsk_put(inet_twsk(sk));
1961                 goto csum_error;
1962         }
1963
1964         switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
1965         case TCP_TW_SYN:
1966         {
1967                 struct sock *sk2;
1968
1969                 sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1970                                             skb, __tcp_hdrlen(th),
1971                                             &ipv6_hdr(skb)->saddr, th->source,
1972                                             &ipv6_hdr(skb)->daddr,
1973                                             ntohs(th->dest),
1974                                             tcp_v6_iif_l3_slave(skb),
1975                                             sdif);
1976                 if (sk2) {
1977                         struct inet_timewait_sock *tw = inet_twsk(sk);
1978                         inet_twsk_deschedule_put(tw);
1979                         sk = sk2;
1980                         tcp_v6_restore_cb(skb);
1981                         refcounted = false;
1982                         goto process;
1983                 }
1984         }
1985                 /* to ACK */
1986                 fallthrough;
1987         case TCP_TW_ACK:
1988                 tcp_v6_timewait_ack(sk, skb);
1989                 break;
1990         case TCP_TW_RST:
1991                 tcp_v6_send_reset(sk, skb);
1992                 inet_twsk_deschedule_put(inet_twsk(sk));
1993                 goto discard_it;
1994         case TCP_TW_SUCCESS:
1995                 ;
1996         }
1997         goto discard_it;
1998 }
1999
2000 void tcp_v6_early_demux(struct sk_buff *skb)
2001 {
2002         struct net *net = dev_net(skb->dev);
2003         const struct ipv6hdr *hdr;
2004         const struct tcphdr *th;
2005         struct sock *sk;
2006
2007         if (skb->pkt_type != PACKET_HOST)
2008                 return;
2009
2010         if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
2011                 return;
2012
2013         hdr = ipv6_hdr(skb);
2014         th = tcp_hdr(skb);
2015
2016         if (th->doff < sizeof(struct tcphdr) / 4)
2017                 return;
2018
2019         /* Note : We use inet6_iif() here, not tcp_v6_iif() */
2020         sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
2021                                         &hdr->saddr, th->source,
2022                                         &hdr->daddr, ntohs(th->dest),
2023                                         inet6_iif(skb), inet6_sdif(skb));
2024         if (sk) {
2025                 skb->sk = sk;
2026                 skb->destructor = sock_edemux;
2027                 if (sk_fullsock(sk)) {
2028                         struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
2029
2030                         if (dst)
2031                                 dst = dst_check(dst, sk->sk_rx_dst_cookie);
2032                         if (dst &&
2033                             sk->sk_rx_dst_ifindex == skb->skb_iif)
2034                                 skb_dst_set_noref(skb, dst);
2035                 }
2036         }
2037 }
2038
2039 static struct timewait_sock_ops tcp6_timewait_sock_ops = {
2040         .twsk_obj_size  = sizeof(struct tcp6_timewait_sock),
2041         .twsk_unique    = tcp_twsk_unique,
2042         .twsk_destructor = tcp_twsk_destructor,
2043 };
2044
2045 INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
2046 {
2047         __tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr);
2048 }
2049
2050 const struct inet_connection_sock_af_ops ipv6_specific = {
2051         .queue_xmit        = inet6_csk_xmit,
2052         .send_check        = tcp_v6_send_check,
2053         .rebuild_header    = inet6_sk_rebuild_header,
2054         .sk_rx_dst_set     = inet6_sk_rx_dst_set,
2055         .conn_request      = tcp_v6_conn_request,
2056         .syn_recv_sock     = tcp_v6_syn_recv_sock,
2057         .net_header_len    = sizeof(struct ipv6hdr),
2058         .setsockopt        = ipv6_setsockopt,
2059         .getsockopt        = ipv6_getsockopt,
2060         .addr2sockaddr     = inet6_csk_addr2sockaddr,
2061         .sockaddr_len      = sizeof(struct sockaddr_in6),
2062         .mtu_reduced       = tcp_v6_mtu_reduced,
2063 };
2064
2065 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2066 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
2067 #ifdef CONFIG_TCP_MD5SIG
2068         .md5_lookup     =       tcp_v6_md5_lookup,
2069         .calc_md5_hash  =       tcp_v6_md5_hash_skb,
2070         .md5_parse      =       tcp_v6_parse_md5_keys,
2071 #endif
2072 #ifdef CONFIG_TCP_AO
2073         .ao_lookup      =       tcp_v6_ao_lookup,
2074         .calc_ao_hash   =       tcp_v6_ao_hash_skb,
2075         .ao_parse       =       tcp_v6_parse_ao,
2076         .ao_calc_key_sk =       tcp_v6_ao_calc_key_sk,
2077 #endif
2078 };
2079 #endif
2080
2081 /*
2082  *      TCP over IPv4 via INET6 API
2083  */
2084 static const struct inet_connection_sock_af_ops ipv6_mapped = {
2085         .queue_xmit        = ip_queue_xmit,
2086         .send_check        = tcp_v4_send_check,
2087         .rebuild_header    = inet_sk_rebuild_header,
2088         .sk_rx_dst_set     = inet_sk_rx_dst_set,
2089         .conn_request      = tcp_v6_conn_request,
2090         .syn_recv_sock     = tcp_v6_syn_recv_sock,
2091         .net_header_len    = sizeof(struct iphdr),
2092         .setsockopt        = ipv6_setsockopt,
2093         .getsockopt        = ipv6_getsockopt,
2094         .addr2sockaddr     = inet6_csk_addr2sockaddr,
2095         .sockaddr_len      = sizeof(struct sockaddr_in6),
2096         .mtu_reduced       = tcp_v4_mtu_reduced,
2097 };
2098
2099 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2100 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
2101 #ifdef CONFIG_TCP_MD5SIG
2102         .md5_lookup     =       tcp_v4_md5_lookup,
2103         .calc_md5_hash  =       tcp_v4_md5_hash_skb,
2104         .md5_parse      =       tcp_v6_parse_md5_keys,
2105 #endif
2106 #ifdef CONFIG_TCP_AO
2107         .ao_lookup      =       tcp_v6_ao_lookup,
2108         .calc_ao_hash   =       tcp_v4_ao_hash_skb,
2109         .ao_parse       =       tcp_v6_parse_ao,
2110         .ao_calc_key_sk =       tcp_v4_ao_calc_key_sk,
2111 #endif
2112 };
2113 #endif
2114
2115 /* NOTE: A lot of things set to zero explicitly by call to
2116  *       sk_alloc() so need not be done here.
2117  */
2118 static int tcp_v6_init_sock(struct sock *sk)
2119 {
2120         struct inet_connection_sock *icsk = inet_csk(sk);
2121
2122         tcp_init_sock(sk);
2123
2124         icsk->icsk_af_ops = &ipv6_specific;
2125
2126 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2127         tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
2128 #endif
2129
2130         return 0;
2131 }
2132
2133 #ifdef CONFIG_PROC_FS
2134 /* Proc filesystem TCPv6 sock list dumping. */
2135 static void get_openreq6(struct seq_file *seq,
2136                          const struct request_sock *req, int i)
2137 {
2138         long ttd = req->rsk_timer.expires - jiffies;
2139         const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
2140         const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;
2141
2142         if (ttd < 0)
2143                 ttd = 0;
2144
2145         seq_printf(seq,
2146                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2147                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
2148                    i,
2149                    src->s6_addr32[0], src->s6_addr32[1],
2150                    src->s6_addr32[2], src->s6_addr32[3],
2151                    inet_rsk(req)->ir_num,
2152                    dest->s6_addr32[0], dest->s6_addr32[1],
2153                    dest->s6_addr32[2], dest->s6_addr32[3],
2154                    ntohs(inet_rsk(req)->ir_rmt_port),
2155                    TCP_SYN_RECV,
2156                    0, 0, /* could print option size, but that is af dependent. */
2157                    1,   /* timers active (only the expire timer) */
2158                    jiffies_to_clock_t(ttd),
2159                    req->num_timeout,
2160                    from_kuid_munged(seq_user_ns(seq),
2161                                     sock_i_uid(req->rsk_listener)),
2162                    0,  /* non standard timer */
2163                    0, /* open_requests have no inode */
2164                    0, req);
2165 }
2166
2167 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
2168 {
2169         const struct in6_addr *dest, *src;
2170         __u16 destp, srcp;
2171         int timer_active;
2172         unsigned long timer_expires;
2173         const struct inet_sock *inet = inet_sk(sp);
2174         const struct tcp_sock *tp = tcp_sk(sp);
2175         const struct inet_connection_sock *icsk = inet_csk(sp);
2176         const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2177         int rx_queue;
2178         int state;
2179
2180         dest  = &sp->sk_v6_daddr;
2181         src   = &sp->sk_v6_rcv_saddr;
2182         destp = ntohs(inet->inet_dport);
2183         srcp  = ntohs(inet->inet_sport);
2184
2185         if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2186             icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2187             icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2188                 timer_active    = 1;
2189                 timer_expires   = icsk->icsk_timeout;
2190         } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2191                 timer_active    = 4;
2192                 timer_expires   = icsk->icsk_timeout;
2193         } else if (timer_pending(&sp->sk_timer)) {
2194                 timer_active    = 2;
2195                 timer_expires   = sp->sk_timer.expires;
2196         } else {
2197                 timer_active    = 0;
2198                 timer_expires = jiffies;
2199         }
2200
2201         state = inet_sk_state_load(sp);
2202         if (state == TCP_LISTEN)
2203                 rx_queue = READ_ONCE(sp->sk_ack_backlog);
2204         else
2205                 /* Because we don't lock the socket,
2206                  * we might find a transient negative value.
2207                  */
2208                 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2209                                       READ_ONCE(tp->copied_seq), 0);
2210
2211         seq_printf(seq,
2212                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2213                    "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
2214                    i,
2215                    src->s6_addr32[0], src->s6_addr32[1],
2216                    src->s6_addr32[2], src->s6_addr32[3], srcp,
2217                    dest->s6_addr32[0], dest->s6_addr32[1],
2218                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
2219                    state,
2220                    READ_ONCE(tp->write_seq) - tp->snd_una,
2221                    rx_queue,
2222                    timer_active,
2223                    jiffies_delta_to_clock_t(timer_expires - jiffies),
2224                    icsk->icsk_retransmits,
2225                    from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
2226                    icsk->icsk_probes_out,
2227                    sock_i_ino(sp),
2228                    refcount_read(&sp->sk_refcnt), sp,
2229                    jiffies_to_clock_t(icsk->icsk_rto),
2230                    jiffies_to_clock_t(icsk->icsk_ack.ato),
2231                    (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
2232                    tcp_snd_cwnd(tp),
2233                    state == TCP_LISTEN ?
2234                         fastopenq->max_qlen :
2235                         (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2236                    );
2237 }
2238
2239 static void get_timewait6_sock(struct seq_file *seq,
2240                                struct inet_timewait_sock *tw, int i)
2241 {
2242         long delta = tw->tw_timer.expires - jiffies;
2243         const struct in6_addr *dest, *src;
2244         __u16 destp, srcp;
2245
2246         dest = &tw->tw_v6_daddr;
2247         src  = &tw->tw_v6_rcv_saddr;
2248         destp = ntohs(tw->tw_dport);
2249         srcp  = ntohs(tw->tw_sport);
2250
2251         seq_printf(seq,
2252                    "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2253                    "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2254                    i,
2255                    src->s6_addr32[0], src->s6_addr32[1],
2256                    src->s6_addr32[2], src->s6_addr32[3], srcp,
2257                    dest->s6_addr32[0], dest->s6_addr32[1],
2258                    dest->s6_addr32[2], dest->s6_addr32[3], destp,
2259                    tw->tw_substate, 0, 0,
2260                    3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2261                    refcount_read(&tw->tw_refcnt), tw);
2262 }
2263
2264 static int tcp6_seq_show(struct seq_file *seq, void *v)
2265 {
2266         struct tcp_iter_state *st;
2267         struct sock *sk = v;
2268
2269         if (v == SEQ_START_TOKEN) {
2270                 seq_puts(seq,
2271                          "  sl  "
2272                          "local_address                         "
2273                          "remote_address                        "
2274                          "st tx_queue rx_queue tr tm->when retrnsmt"
2275                          "   uid  timeout inode\n");
2276                 goto out;
2277         }
2278         st = seq->private;
2279
2280         if (sk->sk_state == TCP_TIME_WAIT)
2281                 get_timewait6_sock(seq, v, st->num);
2282         else if (sk->sk_state == TCP_NEW_SYN_RECV)
2283                 get_openreq6(seq, v, st->num);
2284         else
2285                 get_tcp6_sock(seq, v, st->num);
2286 out:
2287         return 0;
2288 }
2289
2290 static const struct seq_operations tcp6_seq_ops = {
2291         .show           = tcp6_seq_show,
2292         .start          = tcp_seq_start,
2293         .next           = tcp_seq_next,
2294         .stop           = tcp_seq_stop,
2295 };
2296
2297 static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2298         .family         = AF_INET6,
2299 };
2300
2301 int __net_init tcp6_proc_init(struct net *net)
2302 {
2303         if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
2304                         sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
2305                 return -ENOMEM;
2306         return 0;
2307 }
2308
2309 void tcp6_proc_exit(struct net *net)
2310 {
2311         remove_proc_entry("tcp6", net->proc_net);
2312 }
2313 #endif
2314
2315 struct proto tcpv6_prot = {
2316         .name                   = "TCPv6",
2317         .owner                  = THIS_MODULE,
2318         .close                  = tcp_close,
2319         .pre_connect            = tcp_v6_pre_connect,
2320         .connect                = tcp_v6_connect,
2321         .disconnect             = tcp_disconnect,
2322         .accept                 = inet_csk_accept,
2323         .ioctl                  = tcp_ioctl,
2324         .init                   = tcp_v6_init_sock,
2325         .destroy                = tcp_v4_destroy_sock,
2326         .shutdown               = tcp_shutdown,
2327         .setsockopt             = tcp_setsockopt,
2328         .getsockopt             = tcp_getsockopt,
2329         .bpf_bypass_getsockopt  = tcp_bpf_bypass_getsockopt,
2330         .keepalive              = tcp_set_keepalive,
2331         .recvmsg                = tcp_recvmsg,
2332         .sendmsg                = tcp_sendmsg,
2333         .splice_eof             = tcp_splice_eof,
2334         .backlog_rcv            = tcp_v6_do_rcv,
2335         .release_cb             = tcp_release_cb,
2336         .hash                   = inet6_hash,
2337         .unhash                 = inet_unhash,
2338         .get_port               = inet_csk_get_port,
2339         .put_port               = inet_put_port,
2340 #ifdef CONFIG_BPF_SYSCALL
2341         .psock_update_sk_prot   = tcp_bpf_update_proto,
2342 #endif
2343         .enter_memory_pressure  = tcp_enter_memory_pressure,
2344         .leave_memory_pressure  = tcp_leave_memory_pressure,
2345         .stream_memory_free     = tcp_stream_memory_free,
2346         .sockets_allocated      = &tcp_sockets_allocated,
2347
2348         .memory_allocated       = &tcp_memory_allocated,
2349         .per_cpu_fw_alloc       = &tcp_memory_per_cpu_fw_alloc,
2350
2351         .memory_pressure        = &tcp_memory_pressure,
2352         .orphan_count           = &tcp_orphan_count,
2353         .sysctl_mem             = sysctl_tcp_mem,
2354         .sysctl_wmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2355         .sysctl_rmem_offset     = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2356         .max_header             = MAX_TCP_HEADER,
2357         .obj_size               = sizeof(struct tcp6_sock),
2358         .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
2359         .slab_flags             = SLAB_TYPESAFE_BY_RCU,
2360         .twsk_prot              = &tcp6_timewait_sock_ops,
2361         .rsk_prot               = &tcp6_request_sock_ops,
2362         .h.hashinfo             = NULL,
2363         .no_autobind            = true,
2364         .diag_destroy           = tcp_abort,
2365 };
2366 EXPORT_SYMBOL_GPL(tcpv6_prot);
2367
2368 static const struct inet6_protocol tcpv6_protocol = {
2369         .handler        =       tcp_v6_rcv,
2370         .err_handler    =       tcp_v6_err,
2371         .flags          =       INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
2372 };
2373
2374 static struct inet_protosw tcpv6_protosw = {
2375         .type           =       SOCK_STREAM,
2376         .protocol       =       IPPROTO_TCP,
2377         .prot           =       &tcpv6_prot,
2378         .ops            =       &inet6_stream_ops,
2379         .flags          =       INET_PROTOSW_PERMANENT |
2380                                 INET_PROTOSW_ICSK,
2381 };
2382
2383 static int __net_init tcpv6_net_init(struct net *net)
2384 {
2385         return inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
2386                                     SOCK_RAW, IPPROTO_TCP, net);
2387 }
2388
2389 static void __net_exit tcpv6_net_exit(struct net *net)
2390 {
2391         inet_ctl_sock_destroy(net->ipv6.tcp_sk);
2392 }
2393
2394 static void __net_exit tcpv6_net_exit_batch(struct list_head *net_exit_list)
2395 {
2396         tcp_twsk_purge(net_exit_list, AF_INET6);
2397 }
2398
2399 static struct pernet_operations tcpv6_net_ops = {
2400         .init       = tcpv6_net_init,
2401         .exit       = tcpv6_net_exit,
2402         .exit_batch = tcpv6_net_exit_batch,
2403 };
2404
2405 int __init tcpv6_init(void)
2406 {
2407         int ret;
2408
2409         ret = inet6_add_protocol(&tcpv6_protocol, IPPROTO_TCP);
2410         if (ret)
2411                 goto out;
2412
2413         /* register inet6 protocol */
2414         ret = inet6_register_protosw(&tcpv6_protosw);
2415         if (ret)
2416                 goto out_tcpv6_protocol;
2417
2418         ret = register_pernet_subsys(&tcpv6_net_ops);
2419         if (ret)
2420                 goto out_tcpv6_protosw;
2421
2422         ret = mptcpv6_init();
2423         if (ret)
2424                 goto out_tcpv6_pernet_subsys;
2425
2426 out:
2427         return ret;
2428
2429 out_tcpv6_pernet_subsys:
2430         unregister_pernet_subsys(&tcpv6_net_ops);
2431 out_tcpv6_protosw:
2432         inet6_unregister_protosw(&tcpv6_protosw);
2433 out_tcpv6_protocol:
2434         inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2435         goto out;
2436 }
2437
2438 void tcpv6_exit(void)
2439 {
2440         unregister_pernet_subsys(&tcpv6_net_ops);
2441         inet6_unregister_protosw(&tcpv6_protosw);
2442         inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2443 }