net/netfilter/nf_conntrack_core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Connection state tracking for netfilter.  This is separated from,
   3    but required by, the NAT layer; it can also be used by an iptables
   4    extension. */
   5
   6 /* (C) 1999-2001 Paul `Rusty' Russell
   7  * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
   8  * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
   9  * (C) 2005-2012 Patrick McHardy <kaber@trash.net>
  10  */
  11
  12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  13
  14 #include <linux/types.h>
  15 #include <linux/netfilter.h>
  16 #include <linux/module.h>
  17 #include <linux/sched.h>
  18 #include <linux/skbuff.h>
  19 #include <linux/proc_fs.h>
  20 #include <linux/vmalloc.h>
  21 #include <linux/stddef.h>
  22 #include <linux/slab.h>
  23 #include <linux/random.h>
  24 #include <linux/jhash.h>
  25 #include <linux/siphash.h>
  26 #include <linux/err.h>
  27 #include <linux/percpu.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/notifier.h>
  30 #include <linux/kernel.h>
  31 #include <linux/netdevice.h>
  32 #include <linux/socket.h>
  33 #include <linux/mm.h>
  34 #include <linux/nsproxy.h>
  35 #include <linux/rculist_nulls.h>
  36
  37 #include <net/netfilter/nf_conntrack.h>
  38 #include <net/netfilter/nf_conntrack_l4proto.h>
  39 #include <net/netfilter/nf_conntrack_expect.h>
  40 #include <net/netfilter/nf_conntrack_helper.h>
  41 #include <net/netfilter/nf_conntrack_seqadj.h>
  42 #include <net/netfilter/nf_conntrack_core.h>
  43 #include <net/netfilter/nf_conntrack_extend.h>
  44 #include <net/netfilter/nf_conntrack_acct.h>
  45 #include <net/netfilter/nf_conntrack_ecache.h>
  46 #include <net/netfilter/nf_conntrack_zones.h>
  47 #include <net/netfilter/nf_conntrack_timestamp.h>
  48 #include <net/netfilter/nf_conntrack_timeout.h>
  49 #include <net/netfilter/nf_conntrack_labels.h>
  50 #include <net/netfilter/nf_conntrack_synproxy.h>
  51 #include <net/netfilter/nf_nat.h>
  52 #include <net/netfilter/nf_nat_helper.h>
  53 #include <net/netns/hash.h>
  54 #include <net/ip.h>
  55
  56 #include "nf_internals.h"
  57
  58 extern unsigned int nf_conntrack_net_id;
  59
  60 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];
  61 EXPORT_SYMBOL_GPL(nf_conntrack_locks);
  62
  63 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);
  64 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);
  65
  66 struct hlist_nulls_head *nf_conntrack_hash __read_mostly;
  67 EXPORT_SYMBOL_GPL(nf_conntrack_hash);
  68
  69 struct conntrack_gc_work {
  70         struct delayed_work     dwork;
  71         u32                     next_bucket;
  72         bool                    exiting;
  73         bool                    early_drop;
  74 };
  75
  76 static __read_mostly struct kmem_cache *nf_conntrack_cachep;
  77 static DEFINE_SPINLOCK(nf_conntrack_locks_all_lock);
  78 static __read_mostly bool nf_conntrack_locks_all;
  79
  80 #define GC_SCAN_INTERVAL        (120u * HZ)
  81 #define GC_SCAN_MAX_DURATION    msecs_to_jiffies(10)
  82
  83 static struct conntrack_gc_work conntrack_gc_work;
  84
  85 extern unsigned int nf_conntrack_net_id;
  86
  87 void nf_conntrack_lock(spinlock_t *lock) __acquires(lock)
  88 {
  89         /* 1) Acquire the lock */
  90         spin_lock(lock);
  91
  92         /* 2) read nf_conntrack_locks_all, with ACQUIRE semantics
  93          * It pairs with the smp_store_release() in nf_conntrack_all_unlock()
  94          */
  95         if (likely(smp_load_acquire(&nf_conntrack_locks_all) == false))
  96                 return;
  97
  98         /* fast path failed, unlock */
  99         spin_unlock(lock);
 100
 101         /* Slow path 1) get global lock */
 102         spin_lock(&nf_conntrack_locks_all_lock);
 103
 104         /* Slow path 2) get the lock we want */
 105         spin_lock(lock);
 106
 107         /* Slow path 3) release the global lock */
 108         spin_unlock(&nf_conntrack_locks_all_lock);
 109 }
 110 EXPORT_SYMBOL_GPL(nf_conntrack_lock);
 111
 112 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)
 113 {
 114         h1 %= CONNTRACK_LOCKS;
 115         h2 %= CONNTRACK_LOCKS;
 116         spin_unlock(&nf_conntrack_locks[h1]);
 117         if (h1 != h2)
 118                 spin_unlock(&nf_conntrack_locks[h2]);
 119 }
 120
 121 /* return true if we need to recompute hashes (in case hash table was resized) */
 122 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,
 123                                      unsigned int h2, unsigned int sequence)
 124 {
 125         h1 %= CONNTRACK_LOCKS;
 126         h2 %= CONNTRACK_LOCKS;
 127         if (h1 <= h2) {
 128                 nf_conntrack_lock(&nf_conntrack_locks[h1]);
 129                 if (h1 != h2)
 130                         spin_lock_nested(&nf_conntrack_locks[h2],
 131                                          SINGLE_DEPTH_NESTING);
 132         } else {
 133                 nf_conntrack_lock(&nf_conntrack_locks[h2]);
 134                 spin_lock_nested(&nf_conntrack_locks[h1],
 135                                  SINGLE_DEPTH_NESTING);
 136         }
 137         if (read_seqcount_retry(&nf_conntrack_generation, sequence)) {
 138                 nf_conntrack_double_unlock(h1, h2);
 139                 return true;
 140         }
 141         return false;
 142 }
 143
 144 static void nf_conntrack_all_lock(void)
 145         __acquires(&nf_conntrack_locks_all_lock)
 146 {
 147         int i;
 148
 149         spin_lock(&nf_conntrack_locks_all_lock);
 150
 151         nf_conntrack_locks_all = true;
 152
 153         for (i = 0; i < CONNTRACK_LOCKS; i++) {
 154                 spin_lock(&nf_conntrack_locks[i]);
 155
 156                 /* This spin_unlock provides the "release" to ensure that
 157                  * nf_conntrack_locks_all==true is visible to everyone that
 158                  * acquired spin_lock(&nf_conntrack_locks[]).
 159                  */
 160                 spin_unlock(&nf_conntrack_locks[i]);
 161         }
 162 }
 163
 164 static void nf_conntrack_all_unlock(void)
 165         __releases(&nf_conntrack_locks_all_lock)
 166 {
 167         /* All prior stores must be complete before we clear
 168          * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock()
 169          * might observe the false value but not the entire
 170          * critical section.
 171          * It pairs with the smp_load_acquire() in nf_conntrack_lock()
 172          */
 173         smp_store_release(&nf_conntrack_locks_all, false);
 174         spin_unlock(&nf_conntrack_locks_all_lock);
 175 }
 176
 177 unsigned int nf_conntrack_htable_size __read_mostly;
 178 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
 179
 180 unsigned int nf_conntrack_max __read_mostly;
 181 EXPORT_SYMBOL_GPL(nf_conntrack_max);
 182 seqcount_spinlock_t nf_conntrack_generation __read_mostly;
 183 static unsigned int nf_conntrack_hash_rnd __read_mostly;
 184
 185 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple,
 186                               const struct net *net)
 187 {
 188         unsigned int n;
 189         u32 seed;
 190
 191         get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd));
 192
 193         /* The direction must be ignored, so we hash everything up to the
 194          * destination ports (which is a multiple of 4) and treat the last
 195          * three bytes manually.
 196          */
 197         seed = nf_conntrack_hash_rnd ^ net_hash_mix(net);
 198         n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
 199         return jhash2((u32 *)tuple, n, seed ^
 200                       (((__force __u16)tuple->dst.u.all << 16) |
 201                       tuple->dst.protonum));
 202 }
 203
 204 static u32 scale_hash(u32 hash)
 205 {
 206         return reciprocal_scale(hash, nf_conntrack_htable_size);
 207 }
 208
 209 static u32 __hash_conntrack(const struct net *net,
 210                             const struct nf_conntrack_tuple *tuple,
 211                             unsigned int size)
 212 {
 213         return reciprocal_scale(hash_conntrack_raw(tuple, net), size);
 214 }
 215
 216 static u32 hash_conntrack(const struct net *net,
 217                           const struct nf_conntrack_tuple *tuple)
 218 {
 219         return scale_hash(hash_conntrack_raw(tuple, net));
 220 }
 221
 222 static bool nf_ct_get_tuple_ports(const struct sk_buff *skb,
 223                                   unsigned int dataoff,
 224                                   struct nf_conntrack_tuple *tuple)
 225 {       struct {
 226                 __be16 sport;
 227                 __be16 dport;
 228         } _inet_hdr, *inet_hdr;
 229
 230         /* Actually only need first 4 bytes to get ports. */
 231         inet_hdr = skb_header_pointer(skb, dataoff, sizeof(_inet_hdr), &_inet_hdr);
 232         if (!inet_hdr)
 233                 return false;
 234
 235         tuple->src.u.udp.port = inet_hdr->sport;
 236         tuple->dst.u.udp.port = inet_hdr->dport;
 237         return true;
 238 }
 239
 240 static bool
 241 nf_ct_get_tuple(const struct sk_buff *skb,
 242                 unsigned int nhoff,
 243                 unsigned int dataoff,
 244                 u_int16_t l3num,
 245                 u_int8_t protonum,
 246                 struct net *net,
 247                 struct nf_conntrack_tuple *tuple)
 248 {
 249         unsigned int size;
 250         const __be32 *ap;
 251         __be32 _addrs[8];
 252
 253         memset(tuple, 0, sizeof(*tuple));
 254
 255         tuple->src.l3num = l3num;
 256         switch (l3num) {
 257         case NFPROTO_IPV4:
 258                 nhoff += offsetof(struct iphdr, saddr);
 259                 size = 2 * sizeof(__be32);
 260                 break;
 261         case NFPROTO_IPV6:
 262                 nhoff += offsetof(struct ipv6hdr, saddr);
 263                 size = sizeof(_addrs);
 264                 break;
 265         default:
 266                 return true;
 267         }
 268
 269         ap = skb_header_pointer(skb, nhoff, size, _addrs);
 270         if (!ap)
 271                 return false;
 272
 273         switch (l3num) {
 274         case NFPROTO_IPV4:
 275                 tuple->src.u3.ip = ap[0];
 276                 tuple->dst.u3.ip = ap[1];
 277                 break;
 278         case NFPROTO_IPV6:
 279                 memcpy(tuple->src.u3.ip6, ap, sizeof(tuple->src.u3.ip6));
 280                 memcpy(tuple->dst.u3.ip6, ap + 4, sizeof(tuple->dst.u3.ip6));
 281                 break;
 282         }
 283
 284         tuple->dst.protonum = protonum;
 285         tuple->dst.dir = IP_CT_DIR_ORIGINAL;
 286
 287         switch (protonum) {
 288 #if IS_ENABLED(CONFIG_IPV6)
 289         case IPPROTO_ICMPV6:
 290                 return icmpv6_pkt_to_tuple(skb, dataoff, net, tuple);
 291 #endif
 292         case IPPROTO_ICMP:
 293                 return icmp_pkt_to_tuple(skb, dataoff, net, tuple);
 294 #ifdef CONFIG_NF_CT_PROTO_GRE
 295         case IPPROTO_GRE:
 296                 return gre_pkt_to_tuple(skb, dataoff, net, tuple);
 297 #endif
 298         case IPPROTO_TCP:
 299         case IPPROTO_UDP: /* fallthrough */
 300                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 301 #ifdef CONFIG_NF_CT_PROTO_UDPLITE
 302         case IPPROTO_UDPLITE:
 303                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 304 #endif
 305 #ifdef CONFIG_NF_CT_PROTO_SCTP
 306         case IPPROTO_SCTP:
 307                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 308 #endif
 309 #ifdef CONFIG_NF_CT_PROTO_DCCP
 310         case IPPROTO_DCCP:
 311                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
 312 #endif
 313         default:
 314                 break;
 315         }
 316
 317         return true;
 318 }
 319
 320 static int ipv4_get_l4proto(const struct sk_buff *skb, unsigned int nhoff,
 321                             u_int8_t *protonum)
 322 {
 323         int dataoff = -1;
 324         const struct iphdr *iph;
 325         struct iphdr _iph;
 326
 327         iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph);
 328         if (!iph)
 329                 return -1;
 330
 331         /* Conntrack defragments packets, we might still see fragments
 332          * inside ICMP packets though.
 333          */
 334         if (iph->frag_off & htons(IP_OFFSET))
 335                 return -1;
 336
 337         dataoff = nhoff + (iph->ihl << 2);
 338         *protonum = iph->protocol;
 339
 340         /* Check bogus IP headers */
 341         if (dataoff > skb->len) {
 342                 pr_debug("bogus IPv4 packet: nhoff %u, ihl %u, skblen %u\n",
 343                          nhoff, iph->ihl << 2, skb->len);
 344                 return -1;
 345         }
 346         return dataoff;
 347 }
 348
 349 #if IS_ENABLED(CONFIG_IPV6)
 350 static int ipv6_get_l4proto(const struct sk_buff *skb, unsigned int nhoff,
 351                             u8 *protonum)
 352 {
 353         int protoff = -1;
 354         unsigned int extoff = nhoff + sizeof(struct ipv6hdr);
 355         __be16 frag_off;
 356         u8 nexthdr;
 357
 358         if (skb_copy_bits(skb, nhoff + offsetof(struct ipv6hdr, nexthdr),
 359                           &nexthdr, sizeof(nexthdr)) != 0) {
 360                 pr_debug("can't get nexthdr\n");
 361                 return -1;
 362         }
 363         protoff = ipv6_skip_exthdr(skb, extoff, &nexthdr, &frag_off);
 364         /*
 365          * (protoff == skb->len) means the packet has not data, just
 366          * IPv6 and possibly extensions headers, but it is tracked anyway
 367          */
 368         if (protoff < 0 || (frag_off & htons(~0x7)) != 0) {
 369                 pr_debug("can't find proto in pkt\n");
 370                 return -1;
 371         }
 372
 373         *protonum = nexthdr;
 374         return protoff;
 375 }
 376 #endif
 377
 378 static int get_l4proto(const struct sk_buff *skb,
 379                        unsigned int nhoff, u8 pf, u8 *l4num)
 380 {
 381         switch (pf) {
 382         case NFPROTO_IPV4:
 383                 return ipv4_get_l4proto(skb, nhoff, l4num);
 384 #if IS_ENABLED(CONFIG_IPV6)
 385         case NFPROTO_IPV6:
 386                 return ipv6_get_l4proto(skb, nhoff, l4num);
 387 #endif
 388         default:
 389                 *l4num = 0;
 390                 break;
 391         }
 392         return -1;
 393 }
 394
 395 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
 396                        u_int16_t l3num,
 397                        struct net *net, struct nf_conntrack_tuple *tuple)
 398 {
 399         u8 protonum;
 400         int protoff;
 401
 402         protoff = get_l4proto(skb, nhoff, l3num, &protonum);
 403         if (protoff <= 0)
 404                 return false;
 405
 406         return nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple);
 407 }
 408 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
 409
 410 bool
 411 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
 412                    const struct nf_conntrack_tuple *orig)
 413 {
 414         memset(inverse, 0, sizeof(*inverse));
 415
 416         inverse->src.l3num = orig->src.l3num;
 417
 418         switch (orig->src.l3num) {
 419         case NFPROTO_IPV4:
 420                 inverse->src.u3.ip = orig->dst.u3.ip;
 421                 inverse->dst.u3.ip = orig->src.u3.ip;
 422                 break;
 423         case NFPROTO_IPV6:
 424                 inverse->src.u3.in6 = orig->dst.u3.in6;
 425                 inverse->dst.u3.in6 = orig->src.u3.in6;
 426                 break;
 427         default:
 428                 break;
 429         }
 430
 431         inverse->dst.dir = !orig->dst.dir;
 432
 433         inverse->dst.protonum = orig->dst.protonum;
 434
 435         switch (orig->dst.protonum) {
 436         case IPPROTO_ICMP:
 437                 return nf_conntrack_invert_icmp_tuple(inverse, orig);
 438 #if IS_ENABLED(CONFIG_IPV6)
 439         case IPPROTO_ICMPV6:
 440                 return nf_conntrack_invert_icmpv6_tuple(inverse, orig);
 441 #endif
 442         }
 443
 444         inverse->src.u.all = orig->dst.u.all;
 445         inverse->dst.u.all = orig->src.u.all;
 446         return true;
 447 }
 448 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
 449
 450 /* Generate a almost-unique pseudo-id for a given conntrack.
 451  *
 452  * intentionally doesn't re-use any of the seeds used for hash
 453  * table location, we assume id gets exposed to userspace.
 454  *
 455  * Following nf_conn items do not change throughout lifetime
 456  * of the nf_conn:
 457  *
 458  * 1. nf_conn address
 459  * 2. nf_conn->master address (normally NULL)
 460  * 3. the associated net namespace
 461  * 4. the original direction tuple
 462  */
 463 u32 nf_ct_get_id(const struct nf_conn *ct)
 464 {
 465         static __read_mostly siphash_key_t ct_id_seed;
 466         unsigned long a, b, c, d;
 467
 468         net_get_random_once(&ct_id_seed, sizeof(ct_id_seed));
 469
 470         a = (unsigned long)ct;
 471         b = (unsigned long)ct->master;
 472         c = (unsigned long)nf_ct_net(ct);
 473         d = (unsigned long)siphash(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 474                                    sizeof(ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple),
 475                                    &ct_id_seed);
 476 #ifdef CONFIG_64BIT
 477         return siphash_4u64((u64)a, (u64)b, (u64)c, (u64)d, &ct_id_seed);
 478 #else
 479         return siphash_4u32((u32)a, (u32)b, (u32)c, (u32)d, &ct_id_seed);
 480 #endif
 481 }
 482 EXPORT_SYMBOL_GPL(nf_ct_get_id);
 483
 484 static void
 485 clean_from_lists(struct nf_conn *ct)
 486 {
 487         pr_debug("clean_from_lists(%p)\n", ct);
 488         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 489         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
 490
 491         /* Destroy all pending expectations */
 492         nf_ct_remove_expectations(ct);
 493 }
 494
 495 /* must be called with local_bh_disable */
 496 static void nf_ct_add_to_dying_list(struct nf_conn *ct)
 497 {
 498         struct ct_pcpu *pcpu;
 499
 500         /* add this conntrack to the (per cpu) dying list */
 501         ct->cpu = smp_processor_id();
 502         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 503
 504         spin_lock(&pcpu->lock);
 505         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 506                              &pcpu->dying);
 507         spin_unlock(&pcpu->lock);
 508 }
 509
 510 /* must be called with local_bh_disable */
 511 static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct)
 512 {
 513         struct ct_pcpu *pcpu;
 514
 515         /* add this conntrack to the (per cpu) unconfirmed list */
 516         ct->cpu = smp_processor_id();
 517         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 518
 519         spin_lock(&pcpu->lock);
 520         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 521                              &pcpu->unconfirmed);
 522         spin_unlock(&pcpu->lock);
 523 }
 524
 525 /* must be called with local_bh_disable */
 526 static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)
 527 {
 528         struct ct_pcpu *pcpu;
 529
 530         /* We overload first tuple to link into unconfirmed or dying list.*/
 531         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 532
 533         spin_lock(&pcpu->lock);
 534         BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
 535         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 536         spin_unlock(&pcpu->lock);
 537 }
 538
 539 #define NFCT_ALIGN(len) (((len) + NFCT_INFOMASK) & ~NFCT_INFOMASK)
 540
 541 /* Released via destroy_conntrack() */
 542 struct nf_conn *nf_ct_tmpl_alloc(struct net *net,
 543                                  const struct nf_conntrack_zone *zone,
 544                                  gfp_t flags)
 545 {
 546         struct nf_conn *tmpl, *p;
 547
 548         if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK) {
 549                 tmpl = kzalloc(sizeof(*tmpl) + NFCT_INFOMASK, flags);
 550                 if (!tmpl)
 551                         return NULL;
 552
 553                 p = tmpl;
 554                 tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p);
 555                 if (tmpl != p) {
 556                         tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p);
 557                         tmpl->proto.tmpl_padto = (char *)tmpl - (char *)p;
 558                 }
 559         } else {
 560                 tmpl = kzalloc(sizeof(*tmpl), flags);
 561                 if (!tmpl)
 562                         return NULL;
 563         }
 564
 565         tmpl->status = IPS_TEMPLATE;
 566         write_pnet(&tmpl->ct_net, net);
 567         nf_ct_zone_add(tmpl, zone);
 568         atomic_set(&tmpl->ct_general.use, 0);
 569
 570         return tmpl;
 571 }
 572 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc);
 573
 574 void nf_ct_tmpl_free(struct nf_conn *tmpl)
 575 {
 576         nf_ct_ext_destroy(tmpl);
 577
 578         if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK)
 579                 kfree((char *)tmpl - tmpl->proto.tmpl_padto);
 580         else
 581                 kfree(tmpl);
 582 }
 583 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free);
 584
 585 static void destroy_gre_conntrack(struct nf_conn *ct)
 586 {
 587 #ifdef CONFIG_NF_CT_PROTO_GRE
 588         struct nf_conn *master = ct->master;
 589
 590         if (master)
 591                 nf_ct_gre_keymap_destroy(master);
 592 #endif
 593 }
 594
 595 static void
 596 destroy_conntrack(struct nf_conntrack *nfct)
 597 {
 598         struct nf_conn *ct = (struct nf_conn *)nfct;
 599
 600         pr_debug("destroy_conntrack(%p)\n", ct);
 601         WARN_ON(atomic_read(&nfct->use) != 0);
 602
 603         if (unlikely(nf_ct_is_template(ct))) {
 604                 nf_ct_tmpl_free(ct);
 605                 return;
 606         }
 607
 608         if (unlikely(nf_ct_protonum(ct) == IPPROTO_GRE))
 609                 destroy_gre_conntrack(ct);
 610
 611         local_bh_disable();
 612         /* Expectations will have been removed in clean_from_lists,
 613          * except TFTP can create an expectation on the first packet,
 614          * before connection is in the list, so we need to clean here,
 615          * too.
 616          */
 617         nf_ct_remove_expectations(ct);
 618
 619         nf_ct_del_from_dying_or_unconfirmed_list(ct);
 620
 621         local_bh_enable();
 622
 623         if (ct->master)
 624                 nf_ct_put(ct->master);
 625
 626         pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
 627         nf_conntrack_free(ct);
 628 }
 629
 630 static void nf_ct_delete_from_lists(struct nf_conn *ct)
 631 {
 632         struct net *net = nf_ct_net(ct);
 633         unsigned int hash, reply_hash;
 634         unsigned int sequence;
 635
 636         nf_ct_helper_destroy(ct);
 637
 638         local_bh_disable();
 639         do {
 640                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 641                 hash = hash_conntrack(net,
 642                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 643                 reply_hash = hash_conntrack(net,
 644                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 645         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 646
 647         clean_from_lists(ct);
 648         nf_conntrack_double_unlock(hash, reply_hash);
 649
 650         nf_ct_add_to_dying_list(ct);
 651
 652         local_bh_enable();
 653 }
 654
 655 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
 656 {
 657         struct nf_conn_tstamp *tstamp;
 658         struct net *net;
 659
 660         if (test_and_set_bit(IPS_DYING_BIT, &ct->status))
 661                 return false;
 662
 663         tstamp = nf_conn_tstamp_find(ct);
 664         if (tstamp) {
 665                 s32 timeout = ct->timeout - nfct_time_stamp;
 666
 667                 tstamp->stop = ktime_get_real_ns();
 668                 if (timeout < 0)
 669                         tstamp->stop -= jiffies_to_nsecs(-timeout);
 670         }
 671
 672         if (nf_conntrack_event_report(IPCT_DESTROY, ct,
 673                                     portid, report) < 0) {
 674                 /* destroy event was not delivered. nf_ct_put will
 675                  * be done by event cache worker on redelivery.
 676                  */
 677                 nf_ct_delete_from_lists(ct);
 678                 nf_conntrack_ecache_work(nf_ct_net(ct), NFCT_ECACHE_DESTROY_FAIL);
 679                 return false;
 680         }
 681
 682         net = nf_ct_net(ct);
 683         if (nf_conntrack_ecache_dwork_pending(net))
 684                 nf_conntrack_ecache_work(net, NFCT_ECACHE_DESTROY_SENT);
 685         nf_ct_delete_from_lists(ct);
 686         nf_ct_put(ct);
 687         return true;
 688 }
 689 EXPORT_SYMBOL_GPL(nf_ct_delete);
 690
 691 static inline bool
 692 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,
 693                 const struct nf_conntrack_tuple *tuple,
 694                 const struct nf_conntrack_zone *zone,
 695                 const struct net *net)
 696 {
 697         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 698
 699         /* A conntrack can be recreated with the equal tuple,
 700          * so we need to check that the conntrack is confirmed
 701          */
 702         return nf_ct_tuple_equal(tuple, &h->tuple) &&
 703                nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) &&
 704                nf_ct_is_confirmed(ct) &&
 705                net_eq(net, nf_ct_net(ct));
 706 }
 707
 708 static inline bool
 709 nf_ct_match(const struct nf_conn *ct1, const struct nf_conn *ct2)
 710 {
 711         return nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 712                                  &ct2->tuplehash[IP_CT_DIR_ORIGINAL].tuple) &&
 713                nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_REPLY].tuple,
 714                                  &ct2->tuplehash[IP_CT_DIR_REPLY].tuple) &&
 715                nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_ORIGINAL) &&
 716                nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_REPLY) &&
 717                net_eq(nf_ct_net(ct1), nf_ct_net(ct2));
 718 }
 719
 720 /* caller must hold rcu readlock and none of the nf_conntrack_locks */
 721 static void nf_ct_gc_expired(struct nf_conn *ct)
 722 {
 723         if (!atomic_inc_not_zero(&ct->ct_general.use))
 724                 return;
 725
 726         if (nf_ct_should_gc(ct))
 727                 nf_ct_kill(ct);
 728
 729         nf_ct_put(ct);
 730 }
 731
 732 /*
 733  * Warning :
 734  * - Caller must take a reference on returned object
 735  *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
 736  */
 737 static struct nf_conntrack_tuple_hash *
 738 ____nf_conntrack_find(struct net *net, const struct nf_conntrack_zone *zone,
 739                       const struct nf_conntrack_tuple *tuple, u32 hash)
 740 {
 741         struct nf_conntrack_tuple_hash *h;
 742         struct hlist_nulls_head *ct_hash;
 743         struct hlist_nulls_node *n;
 744         unsigned int bucket, hsize;
 745
 746 begin:
 747         nf_conntrack_get_ht(&ct_hash, &hsize);
 748         bucket = reciprocal_scale(hash, hsize);
 749
 750         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) {
 751                 struct nf_conn *ct;
 752
 753                 ct = nf_ct_tuplehash_to_ctrack(h);
 754                 if (nf_ct_is_expired(ct)) {
 755                         nf_ct_gc_expired(ct);
 756                         continue;
 757                 }
 758
 759                 if (nf_ct_key_equal(h, tuple, zone, net))
 760                         return h;
 761         }
 762         /*
 763          * if the nulls value we got at the end of this lookup is
 764          * not the expected one, we must restart lookup.
 765          * We probably met an item that was moved to another chain.
 766          */
 767         if (get_nulls_value(n) != bucket) {
 768                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
 769                 goto begin;
 770         }
 771
 772         return NULL;
 773 }
 774
 775 /* Find a connection corresponding to a tuple. */
 776 static struct nf_conntrack_tuple_hash *
 777 __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 778                         const struct nf_conntrack_tuple *tuple, u32 hash)
 779 {
 780         struct nf_conntrack_tuple_hash *h;
 781         struct nf_conn *ct;
 782
 783         rcu_read_lock();
 784
 785         h = ____nf_conntrack_find(net, zone, tuple, hash);
 786         if (h) {
 787                 /* We have a candidate that matches the tuple we're interested
 788                  * in, try to obtain a reference and re-check tuple
 789                  */
 790                 ct = nf_ct_tuplehash_to_ctrack(h);
 791                 if (likely(atomic_inc_not_zero(&ct->ct_general.use))) {
 792                         if (likely(nf_ct_key_equal(h, tuple, zone, net)))
 793                                 goto found;
 794
 795                         /* TYPESAFE_BY_RCU recycled the candidate */
 796                         nf_ct_put(ct);
 797                 }
 798
 799                 h = NULL;
 800         }
 801 found:
 802         rcu_read_unlock();
 803
 804         return h;
 805 }
 806
 807 struct nf_conntrack_tuple_hash *
 808 nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 809                       const struct nf_conntrack_tuple *tuple)
 810 {
 811         return __nf_conntrack_find_get(net, zone, tuple,
 812                                        hash_conntrack_raw(tuple, net));
 813 }
 814 EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
 815
 816 static void __nf_conntrack_hash_insert(struct nf_conn *ct,
 817                                        unsigned int hash,
 818                                        unsigned int reply_hash)
 819 {
 820         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 821                            &nf_conntrack_hash[hash]);
 822         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
 823                            &nf_conntrack_hash[reply_hash]);
 824 }
 825
 826 int
 827 nf_conntrack_hash_check_insert(struct nf_conn *ct)
 828 {
 829         const struct nf_conntrack_zone *zone;
 830         struct net *net = nf_ct_net(ct);
 831         unsigned int hash, reply_hash;
 832         struct nf_conntrack_tuple_hash *h;
 833         struct hlist_nulls_node *n;
 834         unsigned int sequence;
 835
 836         zone = nf_ct_zone(ct);
 837
 838         local_bh_disable();
 839         do {
 840                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 841                 hash = hash_conntrack(net,
 842                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 843                 reply_hash = hash_conntrack(net,
 844                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 845         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 846
 847         /* See if there's one in the list already, including reverse */
 848         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
 849                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 850                                     zone, net))
 851                         goto out;
 852
 853         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
 854                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 855                                     zone, net))
 856                         goto out;
 857
 858         smp_wmb();
 859         /* The caller holds a reference to this object */
 860         atomic_set(&ct->ct_general.use, 2);
 861         __nf_conntrack_hash_insert(ct, hash, reply_hash);
 862         nf_conntrack_double_unlock(hash, reply_hash);
 863         NF_CT_STAT_INC(net, insert);
 864         local_bh_enable();
 865         return 0;
 866
 867 out:
 868         nf_conntrack_double_unlock(hash, reply_hash);
 869         local_bh_enable();
 870         return -EEXIST;
 871 }
 872 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
 873
 874 void nf_ct_acct_add(struct nf_conn *ct, u32 dir, unsigned int packets,
 875                     unsigned int bytes)
 876 {
 877         struct nf_conn_acct *acct;
 878
 879         acct = nf_conn_acct_find(ct);
 880         if (acct) {
 881                 struct nf_conn_counter *counter = acct->counter;
 882
 883                 atomic64_add(packets, &counter[dir].packets);
 884                 atomic64_add(bytes, &counter[dir].bytes);
 885         }
 886 }
 887 EXPORT_SYMBOL_GPL(nf_ct_acct_add);
 888
 889 static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 890                              const struct nf_conn *loser_ct)
 891 {
 892         struct nf_conn_acct *acct;
 893
 894         acct = nf_conn_acct_find(loser_ct);
 895         if (acct) {
 896                 struct nf_conn_counter *counter = acct->counter;
 897                 unsigned int bytes;
 898
 899                 /* u32 should be fine since we must have seen one packet. */
 900                 bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes);
 901                 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), bytes);
 902         }
 903 }
 904
 905 static void __nf_conntrack_insert_prepare(struct nf_conn *ct)
 906 {
 907         struct nf_conn_tstamp *tstamp;
 908
 909         atomic_inc(&ct->ct_general.use);
 910         ct->status |= IPS_CONFIRMED;
 911
 912         /* set conntrack timestamp, if enabled. */
 913         tstamp = nf_conn_tstamp_find(ct);
 914         if (tstamp)
 915                 tstamp->start = ktime_get_real_ns();
 916 }
 917
 918 /* caller must hold locks to prevent concurrent changes */
 919 static int __nf_ct_resolve_clash(struct sk_buff *skb,
 920                                  struct nf_conntrack_tuple_hash *h)
 921 {
 922         /* This is the conntrack entry already in hashes that won race. */
 923         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 924         enum ip_conntrack_info ctinfo;
 925         struct nf_conn *loser_ct;
 926
 927         loser_ct = nf_ct_get(skb, &ctinfo);
 928
 929         if (nf_ct_is_dying(ct))
 930                 return NF_DROP;
 931
 932         if (((ct->status & IPS_NAT_DONE_MASK) == 0) ||
 933             nf_ct_match(ct, loser_ct)) {
 934                 struct net *net = nf_ct_net(ct);
 935
 936                 nf_conntrack_get(&ct->ct_general);
 937
 938                 nf_ct_acct_merge(ct, ctinfo, loser_ct);
 939                 nf_ct_add_to_dying_list(loser_ct);
 940                 nf_conntrack_put(&loser_ct->ct_general);
 941                 nf_ct_set(skb, ct, ctinfo);
 942
 943                 NF_CT_STAT_INC(net, clash_resolve);
 944                 return NF_ACCEPT;
 945         }
 946
 947         return NF_DROP;
 948 }
 949
 950 /**
 951  * nf_ct_resolve_clash_harder - attempt to insert clashing conntrack entry
 952  *
 953  * @skb: skb that causes the collision
 954  * @repl_idx: hash slot for reply direction
 955  *
 956  * Called when origin or reply direction had a clash.
 957  * The skb can be handled without packet drop provided the reply direction
 958  * is unique or there the existing entry has the identical tuple in both
 959  * directions.
 960  *
 961  * Caller must hold conntrack table locks to prevent concurrent updates.
 962  *
 963  * Returns NF_DROP if the clash could not be handled.
 964  */
 965 static int nf_ct_resolve_clash_harder(struct sk_buff *skb, u32 repl_idx)
 966 {
 967         struct nf_conn *loser_ct = (struct nf_conn *)skb_nfct(skb);
 968         const struct nf_conntrack_zone *zone;
 969         struct nf_conntrack_tuple_hash *h;
 970         struct hlist_nulls_node *n;
 971         struct net *net;
 972
 973         zone = nf_ct_zone(loser_ct);
 974         net = nf_ct_net(loser_ct);
 975
 976         /* Reply direction must never result in a clash, unless both origin
 977          * and reply tuples are identical.
 978          */
 979         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[repl_idx], hnnode) {
 980                 if (nf_ct_key_equal(h,
 981                                     &loser_ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 982                                     zone, net))
 983                         return __nf_ct_resolve_clash(skb, h);
 984         }
 985
 986         /* We want the clashing entry to go away real soon: 1 second timeout. */
 987         loser_ct->timeout = nfct_time_stamp + HZ;
 988
 989         /* IPS_NAT_CLASH removes the entry automatically on the first
 990          * reply.  Also prevents UDP tracker from moving the entry to
 991          * ASSURED state, i.e. the entry can always be evicted under
 992          * pressure.
 993          */
 994         loser_ct->status |= IPS_FIXED_TIMEOUT | IPS_NAT_CLASH;
 995
 996         __nf_conntrack_insert_prepare(loser_ct);
 997
 998         /* fake add for ORIGINAL dir: we want lookups to only find the entry
 999          * already in the table.  This also hides the clashing entry from
1000          * ctnetlink iteration, i.e. conntrack -L won't show them.
1001          */
1002         hlist_nulls_add_fake(&loser_ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
1003
1004         hlist_nulls_add_head_rcu(&loser_ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
1005                                  &nf_conntrack_hash[repl_idx]);
1006
1007         NF_CT_STAT_INC(net, clash_resolve);
1008         return NF_ACCEPT;
1009 }
1010
1011 /**
1012  * nf_ct_resolve_clash - attempt to handle clash without packet drop
1013  *
1014  * @skb: skb that causes the clash
1015  * @h: tuplehash of the clashing entry already in table
1016  * @reply_hash: hash slot for reply direction
1017  *
1018  * A conntrack entry can be inserted to the connection tracking table
1019  * if there is no existing entry with an identical tuple.
1020  *
1021  * If there is one, @skb (and the assocated, unconfirmed conntrack) has
1022  * to be dropped.  In case @skb is retransmitted, next conntrack lookup
1023  * will find the already-existing entry.
1024  *
1025  * The major problem with such packet drop is the extra delay added by
1026  * the packet loss -- it will take some time for a retransmit to occur
1027  * (or the sender to time out when waiting for a reply).
1028  *
1029  * This function attempts to handle the situation without packet drop.
1030  *
1031  * If @skb has no NAT transformation or if the colliding entries are
1032  * exactly the same, only the to-be-confirmed conntrack entry is discarded
1033  * and @skb is associated with the conntrack entry already in the table.
1034  *
1035  * Failing that, the new, unconfirmed conntrack is still added to the table
1036  * provided that the collision only occurs in the ORIGINAL direction.
1037  * The new entry will be added only in the non-clashing REPLY direction,
1038  * so packets in the ORIGINAL direction will continue to match the existing
1039  * entry.  The new entry will also have a fixed timeout so it expires --
1040  * due to the collision, it will only see reply traffic.
1041  *
1042  * Returns NF_DROP if the clash could not be resolved.
1043  */
1044 static __cold noinline int
1045 nf_ct_resolve_clash(struct sk_buff *skb, struct nf_conntrack_tuple_hash *h,
1046                     u32 reply_hash)
1047 {
1048         /* This is the conntrack entry already in hashes that won race. */
1049         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
1050         const struct nf_conntrack_l4proto *l4proto;
1051         enum ip_conntrack_info ctinfo;
1052         struct nf_conn *loser_ct;
1053         struct net *net;
1054         int ret;
1055
1056         loser_ct = nf_ct_get(skb, &ctinfo);
1057         net = nf_ct_net(loser_ct);
1058
1059         l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct));
1060         if (!l4proto->allow_clash)
1061                 goto drop;
1062
1063         ret = __nf_ct_resolve_clash(skb, h);
1064         if (ret == NF_ACCEPT)
1065                 return ret;
1066
1067         ret = nf_ct_resolve_clash_harder(skb, reply_hash);
1068         if (ret == NF_ACCEPT)
1069                 return ret;
1070
1071 drop:
1072         nf_ct_add_to_dying_list(loser_ct);
1073         NF_CT_STAT_INC(net, drop);
1074         NF_CT_STAT_INC(net, insert_failed);
1075         return NF_DROP;
1076 }
1077
1078 /* Confirm a connection given skb; places it in hash table */
1079 int
1080 __nf_conntrack_confirm(struct sk_buff *skb)
1081 {
1082         const struct nf_conntrack_zone *zone;
1083         unsigned int hash, reply_hash;
1084         struct nf_conntrack_tuple_hash *h;
1085         struct nf_conn *ct;
1086         struct nf_conn_help *help;
1087         struct hlist_nulls_node *n;
1088         enum ip_conntrack_info ctinfo;
1089         struct net *net;
1090         unsigned int sequence;
1091         int ret = NF_DROP;
1092
1093         ct = nf_ct_get(skb, &ctinfo);
1094         net = nf_ct_net(ct);
1095
1096         /* ipt_REJECT uses nf_conntrack_attach to attach related
1097            ICMP/TCP RST packets in other direction.  Actual packet
1098            which created connection will be IP_CT_NEW or for an
1099            expected connection, IP_CT_RELATED. */
1100         if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
1101                 return NF_ACCEPT;
1102
1103         zone = nf_ct_zone(ct);
1104         local_bh_disable();
1105
1106         do {
1107                 sequence = read_seqcount_begin(&nf_conntrack_generation);
1108                 /* reuse the hash saved before */
1109                 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
1110                 hash = scale_hash(hash);
1111                 reply_hash = hash_conntrack(net,
1112                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
1113
1114         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
1115
1116         /* We're not in hash table, and we refuse to set up related
1117          * connections for unconfirmed conns.  But packet copies and
1118          * REJECT will give spurious warnings here.
1119          */
1120
1121         /* Another skb with the same unconfirmed conntrack may
1122          * win the race. This may happen for bridge(br_flood)
1123          * or broadcast/multicast packets do skb_clone with
1124          * unconfirmed conntrack.
1125          */
1126         if (unlikely(nf_ct_is_confirmed(ct))) {
1127                 WARN_ON_ONCE(1);
1128                 nf_conntrack_double_unlock(hash, reply_hash);
1129                 local_bh_enable();
1130                 return NF_DROP;
1131         }
1132
1133         pr_debug("Confirming conntrack %p\n", ct);
1134         /* We have to check the DYING flag after unlink to prevent
1135          * a race against nf_ct_get_next_corpse() possibly called from
1136          * user context, else we insert an already 'dead' hash, blocking
1137          * further use of that particular connection -JM.
1138          */
1139         nf_ct_del_from_dying_or_unconfirmed_list(ct);
1140
1141         if (unlikely(nf_ct_is_dying(ct))) {
1142                 nf_ct_add_to_dying_list(ct);
1143                 NF_CT_STAT_INC(net, insert_failed);
1144                 goto dying;
1145         }
1146
1147         /* See if there's one in the list already, including reverse:
1148            NAT could have grabbed it without realizing, since we're
1149            not in the hash.  If there is, we lost race. */
1150         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
1151                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
1152                                     zone, net))
1153                         goto out;
1154
1155         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
1156                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
1157                                     zone, net))
1158                         goto out;
1159
1160         /* Timer relative to confirmation time, not original
1161            setting time, otherwise we'd get timer wrap in
1162            weird delay cases. */
1163         ct->timeout += nfct_time_stamp;
1164
1165         __nf_conntrack_insert_prepare(ct);
1166
1167         /* Since the lookup is lockless, hash insertion must be done after
1168          * starting the timer and setting the CONFIRMED bit. The RCU barriers
1169          * guarantee that no other CPU can find the conntrack before the above
1170          * stores are visible.
1171          */
1172         __nf_conntrack_hash_insert(ct, hash, reply_hash);
1173         nf_conntrack_double_unlock(hash, reply_hash);
1174         local_bh_enable();
1175
1176         help = nfct_help(ct);
1177         if (help && help->helper)
1178                 nf_conntrack_event_cache(IPCT_HELPER, ct);
1179
1180         nf_conntrack_event_cache(master_ct(ct) ?
1181                                  IPCT_RELATED : IPCT_NEW, ct);
1182         return NF_ACCEPT;
1183
1184 out:
1185         ret = nf_ct_resolve_clash(skb, h, reply_hash);
1186 dying:
1187         nf_conntrack_double_unlock(hash, reply_hash);
1188         local_bh_enable();
1189         return ret;
1190 }
1191 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
1192
1193 /* Returns true if a connection correspondings to the tuple (required
1194    for NAT). */
1195 int
1196 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
1197                          const struct nf_conn *ignored_conntrack)
1198 {
1199         struct net *net = nf_ct_net(ignored_conntrack);
1200         const struct nf_conntrack_zone *zone;
1201         struct nf_conntrack_tuple_hash *h;
1202         struct hlist_nulls_head *ct_hash;
1203         unsigned int hash, hsize;
1204         struct hlist_nulls_node *n;
1205         struct nf_conn *ct;
1206
1207         zone = nf_ct_zone(ignored_conntrack);
1208
1209         rcu_read_lock();
1210  begin:
1211         nf_conntrack_get_ht(&ct_hash, &hsize);
1212         hash = __hash_conntrack(net, tuple, hsize);
1213
1214         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) {
1215                 ct = nf_ct_tuplehash_to_ctrack(h);
1216
1217                 if (ct == ignored_conntrack)
1218                         continue;
1219
1220                 if (nf_ct_is_expired(ct)) {
1221                         nf_ct_gc_expired(ct);
1222                         continue;
1223                 }
1224
1225                 if (nf_ct_key_equal(h, tuple, zone, net)) {
1226                         /* Tuple is taken already, so caller will need to find
1227                          * a new source port to use.
1228                          *
1229                          * Only exception:
1230                          * If the *original tuples* are identical, then both
1231                          * conntracks refer to the same flow.
1232                          * This is a rare situation, it can occur e.g. when
1233                          * more than one UDP packet is sent from same socket
1234                          * in different threads.
1235                          *
1236                          * Let nf_ct_resolve_clash() deal with this later.
1237                          */
1238                         if (nf_ct_tuple_equal(&ignored_conntrack->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
1239                                               &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple) &&
1240                                               nf_ct_zone_equal(ct, zone, IP_CT_DIR_ORIGINAL))
1241                                 continue;
1242
1243                         NF_CT_STAT_INC_ATOMIC(net, found);
1244                         rcu_read_unlock();
1245                         return 1;
1246                 }
1247         }
1248
1249         if (get_nulls_value(n) != hash) {
1250                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
1251                 goto begin;
1252         }
1253
1254         rcu_read_unlock();
1255
1256         return 0;
1257 }
1258 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
1259
1260 #define NF_CT_EVICTION_RANGE    8
1261
1262 /* There's a small race here where we may free a just-assured
1263    connection.  Too bad: we're in trouble anyway. */
1264 static unsigned int early_drop_list(struct net *net,
1265                                     struct hlist_nulls_head *head)
1266 {
1267         struct nf_conntrack_tuple_hash *h;
1268         struct hlist_nulls_node *n;
1269         unsigned int drops = 0;
1270         struct nf_conn *tmp;
1271
1272         hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) {
1273                 tmp = nf_ct_tuplehash_to_ctrack(h);
1274
1275                 if (test_bit(IPS_OFFLOAD_BIT, &tmp->status))
1276                         continue;
1277
1278                 if (nf_ct_is_expired(tmp)) {
1279                         nf_ct_gc_expired(tmp);
1280                         continue;
1281                 }
1282
1283                 if (test_bit(IPS_ASSURED_BIT, &tmp->status) ||
1284                     !net_eq(nf_ct_net(tmp), net) ||
1285                     nf_ct_is_dying(tmp))
1286                         continue;
1287
1288                 if (!atomic_inc_not_zero(&tmp->ct_general.use))
1289                         continue;
1290
1291                 /* kill only if still in same netns -- might have moved due to
1292                  * SLAB_TYPESAFE_BY_RCU rules.
1293                  *
1294                  * We steal the timer reference.  If that fails timer has
1295                  * already fired or someone else deleted it. Just drop ref
1296                  * and move to next entry.
1297                  */
1298                 if (net_eq(nf_ct_net(tmp), net) &&
1299                     nf_ct_is_confirmed(tmp) &&
1300                     nf_ct_delete(tmp, 0, 0))
1301                         drops++;
1302
1303                 nf_ct_put(tmp);
1304         }
1305
1306         return drops;
1307 }
1308
1309 static noinline int early_drop(struct net *net, unsigned int hash)
1310 {
1311         unsigned int i, bucket;
1312
1313         for (i = 0; i < NF_CT_EVICTION_RANGE; i++) {
1314                 struct hlist_nulls_head *ct_hash;
1315                 unsigned int hsize, drops;
1316
1317                 rcu_read_lock();
1318                 nf_conntrack_get_ht(&ct_hash, &hsize);
1319                 if (!i)
1320                         bucket = reciprocal_scale(hash, hsize);
1321                 else
1322                         bucket = (bucket + 1) % hsize;
1323
1324                 drops = early_drop_list(net, &ct_hash[bucket]);
1325                 rcu_read_unlock();
1326
1327                 if (drops) {
1328                         NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops);
1329                         return true;
1330                 }
1331         }
1332
1333         return false;
1334 }
1335
1336 static bool gc_worker_skip_ct(const struct nf_conn *ct)
1337 {
1338         return !nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct);
1339 }
1340
1341 static bool gc_worker_can_early_drop(const struct nf_conn *ct)
1342 {
1343         const struct nf_conntrack_l4proto *l4proto;
1344
1345         if (!test_bit(IPS_ASSURED_BIT, &ct->status))
1346                 return true;
1347
1348         l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct));
1349         if (l4proto->can_early_drop && l4proto->can_early_drop(ct))
1350                 return true;
1351
1352         return false;
1353 }
1354
1355 static void gc_worker(struct work_struct *work)
1356 {
1357         unsigned long end_time = jiffies + GC_SCAN_MAX_DURATION;
1358         unsigned int i, hashsz, nf_conntrack_max95 = 0;
1359         unsigned long next_run = GC_SCAN_INTERVAL;
1360         struct conntrack_gc_work *gc_work;
1361         gc_work = container_of(work, struct conntrack_gc_work, dwork.work);
1362
1363         i = gc_work->next_bucket;
1364         if (gc_work->early_drop)
1365                 nf_conntrack_max95 = nf_conntrack_max / 100u * 95u;
1366
1367         do {
1368                 struct nf_conntrack_tuple_hash *h;
1369                 struct hlist_nulls_head *ct_hash;
1370                 struct hlist_nulls_node *n;
1371                 struct nf_conn *tmp;
1372
1373                 rcu_read_lock();
1374
1375                 nf_conntrack_get_ht(&ct_hash, &hashsz);
1376                 if (i >= hashsz) {
1377                         rcu_read_unlock();
1378                         break;
1379                 }
1380
1381                 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) {
1382                         struct nf_conntrack_net *cnet;
1383                         struct net *net;
1384
1385                         tmp = nf_ct_tuplehash_to_ctrack(h);
1386
1387                         if (test_bit(IPS_OFFLOAD_BIT, &tmp->status)) {
1388                                 nf_ct_offload_timeout(tmp);
1389                                 continue;
1390                         }
1391
1392                         if (nf_ct_is_expired(tmp)) {
1393                                 nf_ct_gc_expired(tmp);
1394                                 continue;
1395                         }
1396
1397                         if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp))
1398                                 continue;
1399
1400                         net = nf_ct_net(tmp);
1401                         cnet = net_generic(net, nf_conntrack_net_id);
1402                         if (atomic_read(&cnet->count) < nf_conntrack_max95)
1403                                 continue;
1404
1405                         /* need to take reference to avoid possible races */
1406                         if (!atomic_inc_not_zero(&tmp->ct_general.use))
1407                                 continue;
1408
1409                         if (gc_worker_skip_ct(tmp)) {
1410                                 nf_ct_put(tmp);
1411                                 continue;
1412                         }
1413
1414                         if (gc_worker_can_early_drop(tmp))
1415                                 nf_ct_kill(tmp);
1416
1417                         nf_ct_put(tmp);
1418                 }
1419
1420                 /* could check get_nulls_value() here and restart if ct
1421                  * was moved to another chain.  But given gc is best-effort
1422                  * we will just continue with next hash slot.
1423                  */
1424                 rcu_read_unlock();
1425                 cond_resched();
1426                 i++;
1427
1428                 if (time_after(jiffies, end_time) && i < hashsz) {
1429                         gc_work->next_bucket = i;
1430                         next_run = 0;
1431                         break;
1432                 }
1433         } while (i < hashsz);
1434
1435         if (gc_work->exiting)
1436                 return;
1437
1438         /*
1439          * Eviction will normally happen from the packet path, and not
1440          * from this gc worker.
1441          *
1442          * This worker is only here to reap expired entries when system went
1443          * idle after a busy period.
1444          */
1445         if (next_run) {
1446                 gc_work->early_drop = false;
1447                 gc_work->next_bucket = 0;
1448         }
1449         queue_delayed_work(system_power_efficient_wq, &gc_work->dwork, next_run);
1450 }
1451
1452 static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work)
1453 {
1454         INIT_DEFERRABLE_WORK(&gc_work->dwork, gc_worker);
1455         gc_work->exiting = false;
1456 }
1457
1458 static struct nf_conn *
1459 __nf_conntrack_alloc(struct net *net,
1460                      const struct nf_conntrack_zone *zone,
1461                      const struct nf_conntrack_tuple *orig,
1462                      const struct nf_conntrack_tuple *repl,
1463                      gfp_t gfp, u32 hash)
1464 {
1465         struct nf_conntrack_net *cnet = net_generic(net, nf_conntrack_net_id);
1466         unsigned int ct_count;
1467         struct nf_conn *ct;
1468
1469         /* We don't want any race condition at early drop stage */
1470         ct_count = atomic_inc_return(&cnet->count);
1471
1472         if (nf_conntrack_max && unlikely(ct_count > nf_conntrack_max)) {
1473                 if (!early_drop(net, hash)) {
1474                         if (!conntrack_gc_work.early_drop)
1475                                 conntrack_gc_work.early_drop = true;
1476                         atomic_dec(&cnet->count);
1477                         net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
1478                         return ERR_PTR(-ENOMEM);
1479                 }
1480         }
1481
1482         /*
1483          * Do not use kmem_cache_zalloc(), as this cache uses
1484          * SLAB_TYPESAFE_BY_RCU.
1485          */
1486         ct = kmem_cache_alloc(nf_conntrack_cachep, gfp);
1487         if (ct == NULL)
1488                 goto out;
1489
1490         spin_lock_init(&ct->lock);
1491         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
1492         ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
1493         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
1494         /* save hash for reusing when confirming */
1495         *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
1496         ct->status = 0;
1497         ct->timeout = 0;
1498         write_pnet(&ct->ct_net, net);
1499         memset(&ct->__nfct_init_offset, 0,
1500                offsetof(struct nf_conn, proto) -
1501                offsetof(struct nf_conn, __nfct_init_offset));
1502
1503         nf_ct_zone_add(ct, zone);
1504
1505         /* Because we use RCU lookups, we set ct_general.use to zero before
1506          * this is inserted in any list.
1507          */
1508         atomic_set(&ct->ct_general.use, 0);
1509         return ct;
1510 out:
1511         atomic_dec(&cnet->count);
1512         return ERR_PTR(-ENOMEM);
1513 }
1514
1515 struct nf_conn *nf_conntrack_alloc(struct net *net,
1516                                    const struct nf_conntrack_zone *zone,
1517                                    const struct nf_conntrack_tuple *orig,
1518                                    const struct nf_conntrack_tuple *repl,
1519                                    gfp_t gfp)
1520 {
1521         return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
1522 }
1523 EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
1524
1525 void nf_conntrack_free(struct nf_conn *ct)
1526 {
1527         struct net *net = nf_ct_net(ct);
1528         struct nf_conntrack_net *cnet;
1529
1530         /* A freed object has refcnt == 0, that's
1531          * the golden rule for SLAB_TYPESAFE_BY_RCU
1532          */
1533         WARN_ON(atomic_read(&ct->ct_general.use) != 0);
1534
1535         nf_ct_ext_destroy(ct);
1536         kmem_cache_free(nf_conntrack_cachep, ct);
1537         cnet = net_generic(net, nf_conntrack_net_id);
1538
1539         smp_mb__before_atomic();
1540         atomic_dec(&cnet->count);
1541 }
1542 EXPORT_SYMBOL_GPL(nf_conntrack_free);
1543
1544
1545 /* Allocate a new conntrack: we return -ENOMEM if classification
1546    failed due to stress.  Otherwise it really is unclassifiable. */
1547 static noinline struct nf_conntrack_tuple_hash *
1548 init_conntrack(struct net *net, struct nf_conn *tmpl,
1549                const struct nf_conntrack_tuple *tuple,
1550                struct sk_buff *skb,
1551                unsigned int dataoff, u32 hash)
1552 {
1553         struct nf_conn *ct;
1554         struct nf_conn_help *help;
1555         struct nf_conntrack_tuple repl_tuple;
1556         struct nf_conntrack_ecache *ecache;
1557         struct nf_conntrack_expect *exp = NULL;
1558         const struct nf_conntrack_zone *zone;
1559         struct nf_conn_timeout *timeout_ext;
1560         struct nf_conntrack_zone tmp;
1561         struct nf_conntrack_net *cnet;
1562
1563         if (!nf_ct_invert_tuple(&repl_tuple, tuple)) {
1564                 pr_debug("Can't invert tuple.\n");
1565                 return NULL;
1566         }
1567
1568         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1569         ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
1570                                   hash);
1571         if (IS_ERR(ct))
1572                 return (struct nf_conntrack_tuple_hash *)ct;
1573
1574         if (!nf_ct_add_synproxy(ct, tmpl)) {
1575                 nf_conntrack_free(ct);
1576                 return ERR_PTR(-ENOMEM);
1577         }
1578
1579         timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
1580
1581         if (timeout_ext)
1582                 nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout),
1583                                       GFP_ATOMIC);
1584
1585         nf_ct_acct_ext_add(ct, GFP_ATOMIC);
1586         nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
1587         nf_ct_labels_ext_add(ct);
1588
1589         ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
1590         nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
1591                                  ecache ? ecache->expmask : 0,
1592                              GFP_ATOMIC);
1593
1594         local_bh_disable();
1595         cnet = net_generic(net, nf_conntrack_net_id);
1596         if (cnet->expect_count) {
1597                 spin_lock(&nf_conntrack_expect_lock);
1598                 exp = nf_ct_find_expectation(net, zone, tuple);
1599                 if (exp) {
1600                         pr_debug("expectation arrives ct=%p exp=%p\n",
1601                                  ct, exp);
1602                         /* Welcome, Mr. Bond.  We've been expecting you... */
1603                         __set_bit(IPS_EXPECTED_BIT, &ct->status);
1604                         /* exp->master safe, refcnt bumped in nf_ct_find_expectation */
1605                         ct->master = exp->master;
1606                         if (exp->helper) {
1607                                 help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
1608                                 if (help)
1609                                         rcu_assign_pointer(help->helper, exp->helper);
1610                         }
1611
1612 #ifdef CONFIG_NF_CONNTRACK_MARK
1613                         ct->mark = exp->master->mark;
1614 #endif
1615 #ifdef CONFIG_NF_CONNTRACK_SECMARK
1616                         ct->secmark = exp->master->secmark;
1617 #endif
1618                         NF_CT_STAT_INC(net, expect_new);
1619                 }
1620                 spin_unlock(&nf_conntrack_expect_lock);
1621         }
1622         if (!exp)
1623                 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
1624
1625         /* Now it is inserted into the unconfirmed list, bump refcount */
1626         nf_conntrack_get(&ct->ct_general);
1627         nf_ct_add_to_unconfirmed_list(ct);
1628
1629         local_bh_enable();
1630
1631         if (exp) {
1632                 if (exp->expectfn)
1633                         exp->expectfn(ct, exp);
1634                 nf_ct_expect_put(exp);
1635         }
1636
1637         return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
1638 }
1639
1640 /* On success, returns 0, sets skb->_nfct | ctinfo */
1641 static int
1642 resolve_normal_ct(struct nf_conn *tmpl,
1643                   struct sk_buff *skb,
1644                   unsigned int dataoff,
1645                   u_int8_t protonum,
1646                   const struct nf_hook_state *state)
1647 {
1648         const struct nf_conntrack_zone *zone;
1649         struct nf_conntrack_tuple tuple;
1650         struct nf_conntrack_tuple_hash *h;
1651         enum ip_conntrack_info ctinfo;
1652         struct nf_conntrack_zone tmp;
1653         struct nf_conn *ct;
1654         u32 hash;
1655
1656         if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
1657                              dataoff, state->pf, protonum, state->net,
1658                              &tuple)) {
1659                 pr_debug("Can't get tuple\n");
1660                 return 0;
1661         }
1662
1663         /* look for tuple match */
1664         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1665         hash = hash_conntrack_raw(&tuple, state->net);
1666         h = __nf_conntrack_find_get(state->net, zone, &tuple, hash);
1667         if (!h) {
1668                 h = init_conntrack(state->net, tmpl, &tuple,
1669                                    skb, dataoff, hash);
1670                 if (!h)
1671                         return 0;
1672                 if (IS_ERR(h))
1673                         return PTR_ERR(h);
1674         }
1675         ct = nf_ct_tuplehash_to_ctrack(h);
1676
1677         /* It exists; we have (non-exclusive) reference. */
1678         if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
1679                 ctinfo = IP_CT_ESTABLISHED_REPLY;
1680         } else {
1681                 /* Once we've had two way comms, always ESTABLISHED. */
1682                 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
1683                         pr_debug("normal packet for %p\n", ct);
1684                         ctinfo = IP_CT_ESTABLISHED;
1685                 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
1686                         pr_debug("related packet for %p\n", ct);
1687                         ctinfo = IP_CT_RELATED;
1688                 } else {
1689                         pr_debug("new packet for %p\n", ct);
1690                         ctinfo = IP_CT_NEW;
1691                 }
1692         }
1693         nf_ct_set(skb, ct, ctinfo);
1694         return 0;
1695 }
1696
1697 /*
1698  * icmp packets need special treatment to handle error messages that are
1699  * related to a connection.
1700  *
1701  * Callers need to check if skb has a conntrack assigned when this
1702  * helper returns; in such case skb belongs to an already known connection.
1703  */
1704 static unsigned int __cold
1705 nf_conntrack_handle_icmp(struct nf_conn *tmpl,
1706                          struct sk_buff *skb,
1707                          unsigned int dataoff,
1708                          u8 protonum,
1709                          const struct nf_hook_state *state)
1710 {
1711         int ret;
1712
1713         if (state->pf == NFPROTO_IPV4 && protonum == IPPROTO_ICMP)
1714                 ret = nf_conntrack_icmpv4_error(tmpl, skb, dataoff, state);
1715 #if IS_ENABLED(CONFIG_IPV6)
1716         else if (state->pf == NFPROTO_IPV6 && protonum == IPPROTO_ICMPV6)
1717                 ret = nf_conntrack_icmpv6_error(tmpl, skb, dataoff, state);
1718 #endif
1719         else
1720                 return NF_ACCEPT;
1721
1722         if (ret <= 0)
1723                 NF_CT_STAT_INC_ATOMIC(state->net, error);
1724
1725         return ret;
1726 }
1727
1728 static int generic_packet(struct nf_conn *ct, struct sk_buff *skb,
1729                           enum ip_conntrack_info ctinfo)
1730 {
1731         const unsigned int *timeout = nf_ct_timeout_lookup(ct);
1732
1733         if (!timeout)
1734                 timeout = &nf_generic_pernet(nf_ct_net(ct))->timeout;
1735
1736         nf_ct_refresh_acct(ct, ctinfo, skb, *timeout);
1737         return NF_ACCEPT;
1738 }
1739
1740 /* Returns verdict for packet, or -1 for invalid. */
1741 static int nf_conntrack_handle_packet(struct nf_conn *ct,
1742                                       struct sk_buff *skb,
1743                                       unsigned int dataoff,
1744                                       enum ip_conntrack_info ctinfo,
1745                                       const struct nf_hook_state *state)
1746 {
1747         switch (nf_ct_protonum(ct)) {
1748         case IPPROTO_TCP:
1749                 return nf_conntrack_tcp_packet(ct, skb, dataoff,
1750                                                ctinfo, state);
1751         case IPPROTO_UDP:
1752                 return nf_conntrack_udp_packet(ct, skb, dataoff,
1753                                                ctinfo, state);
1754         case IPPROTO_ICMP:
1755                 return nf_conntrack_icmp_packet(ct, skb, ctinfo, state);
1756 #if IS_ENABLED(CONFIG_IPV6)
1757         case IPPROTO_ICMPV6:
1758                 return nf_conntrack_icmpv6_packet(ct, skb, ctinfo, state);
1759 #endif
1760 #ifdef CONFIG_NF_CT_PROTO_UDPLITE
1761         case IPPROTO_UDPLITE:
1762                 return nf_conntrack_udplite_packet(ct, skb, dataoff,
1763                                                    ctinfo, state);
1764 #endif
1765 #ifdef CONFIG_NF_CT_PROTO_SCTP
1766         case IPPROTO_SCTP:
1767                 return nf_conntrack_sctp_packet(ct, skb, dataoff,
1768                                                 ctinfo, state);
1769 #endif
1770 #ifdef CONFIG_NF_CT_PROTO_DCCP
1771         case IPPROTO_DCCP:
1772                 return nf_conntrack_dccp_packet(ct, skb, dataoff,
1773                                                 ctinfo, state);
1774 #endif
1775 #ifdef CONFIG_NF_CT_PROTO_GRE
1776         case IPPROTO_GRE:
1777                 return nf_conntrack_gre_packet(ct, skb, dataoff,
1778                                                ctinfo, state);
1779 #endif
1780         }
1781
1782         return generic_packet(ct, skb, ctinfo);
1783 }
1784
1785 unsigned int
1786 nf_conntrack_in(struct sk_buff *skb, const struct nf_hook_state *state)
1787 {
1788         enum ip_conntrack_info ctinfo;
1789         struct nf_conn *ct, *tmpl;
1790         u_int8_t protonum;
1791         int dataoff, ret;
1792
1793         tmpl = nf_ct_get(skb, &ctinfo);
1794         if (tmpl || ctinfo == IP_CT_UNTRACKED) {
1795                 /* Previously seen (loopback or untracked)?  Ignore. */
1796                 if ((tmpl && !nf_ct_is_template(tmpl)) ||
1797                      ctinfo == IP_CT_UNTRACKED)
1798                         return NF_ACCEPT;
1799                 skb->_nfct = 0;
1800         }
1801
1802         /* rcu_read_lock()ed by nf_hook_thresh */
1803         dataoff = get_l4proto(skb, skb_network_offset(skb), state->pf, &protonum);
1804         if (dataoff <= 0) {
1805                 pr_debug("not prepared to track yet or error occurred\n");
1806                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1807                 ret = NF_ACCEPT;
1808                 goto out;
1809         }
1810
1811         if (protonum == IPPROTO_ICMP || protonum == IPPROTO_ICMPV6) {
1812                 ret = nf_conntrack_handle_icmp(tmpl, skb, dataoff,
1813                                                protonum, state);
1814                 if (ret <= 0) {
1815                         ret = -ret;
1816                         goto out;
1817                 }
1818                 /* ICMP[v6] protocol trackers may assign one conntrack. */
1819                 if (skb->_nfct)
1820                         goto out;
1821         }
1822 repeat:
1823         ret = resolve_normal_ct(tmpl, skb, dataoff,
1824                                 protonum, state);
1825         if (ret < 0) {
1826                 /* Too stressed to deal. */
1827                 NF_CT_STAT_INC_ATOMIC(state->net, drop);
1828                 ret = NF_DROP;
1829                 goto out;
1830         }
1831
1832         ct = nf_ct_get(skb, &ctinfo);
1833         if (!ct) {
1834                 /* Not valid part of a connection */
1835                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1836                 ret = NF_ACCEPT;
1837                 goto out;
1838         }
1839
1840         ret = nf_conntrack_handle_packet(ct, skb, dataoff, ctinfo, state);
1841         if (ret <= 0) {
1842                 /* Invalid: inverse of the return code tells
1843                  * the netfilter core what to do */
1844                 pr_debug("nf_conntrack_in: Can't track with proto module\n");
1845                 nf_conntrack_put(&ct->ct_general);
1846                 skb->_nfct = 0;
1847                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1848                 if (ret == -NF_DROP)
1849                         NF_CT_STAT_INC_ATOMIC(state->net, drop);
1850                 /* Special case: TCP tracker reports an attempt to reopen a
1851                  * closed/aborted connection. We have to go back and create a
1852                  * fresh conntrack.
1853                  */
1854                 if (ret == -NF_REPEAT)
1855                         goto repeat;
1856                 ret = -ret;
1857                 goto out;
1858         }
1859
1860         if (ctinfo == IP_CT_ESTABLISHED_REPLY &&
1861             !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1862                 nf_conntrack_event_cache(IPCT_REPLY, ct);
1863 out:
1864         if (tmpl)
1865                 nf_ct_put(tmpl);
1866
1867         return ret;
1868 }
1869 EXPORT_SYMBOL_GPL(nf_conntrack_in);
1870
1871 /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
1872    implicitly racy: see __nf_conntrack_confirm */
1873 void nf_conntrack_alter_reply(struct nf_conn *ct,
1874                               const struct nf_conntrack_tuple *newreply)
1875 {
1876         struct nf_conn_help *help = nfct_help(ct);
1877
1878         /* Should be unconfirmed, so not in hash table yet */
1879         WARN_ON(nf_ct_is_confirmed(ct));
1880
1881         pr_debug("Altering reply tuple of %p to ", ct);
1882         nf_ct_dump_tuple(newreply);
1883
1884         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1885         if (ct->master || (help && !hlist_empty(&help->expectations)))
1886                 return;
1887
1888         rcu_read_lock();
1889         __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1890         rcu_read_unlock();
1891 }
1892 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1893
1894 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1895 void __nf_ct_refresh_acct(struct nf_conn *ct,
1896                           enum ip_conntrack_info ctinfo,
1897                           const struct sk_buff *skb,
1898                           u32 extra_jiffies,
1899                           bool do_acct)
1900 {
1901         /* Only update if this is not a fixed timeout */
1902         if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1903                 goto acct;
1904
1905         /* If not in hash table, timer will not be active yet */
1906         if (nf_ct_is_confirmed(ct))
1907                 extra_jiffies += nfct_time_stamp;
1908
1909         if (READ_ONCE(ct->timeout) != extra_jiffies)
1910                 WRITE_ONCE(ct->timeout, extra_jiffies);
1911 acct:
1912         if (do_acct)
1913                 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
1914 }
1915 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1916
1917 bool nf_ct_kill_acct(struct nf_conn *ct,
1918                      enum ip_conntrack_info ctinfo,
1919                      const struct sk_buff *skb)
1920 {
1921         nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
1922
1923         return nf_ct_delete(ct, 0, 0);
1924 }
1925 EXPORT_SYMBOL_GPL(nf_ct_kill_acct);
1926
1927 #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1928
1929 #include <linux/netfilter/nfnetlink.h>
1930 #include <linux/netfilter/nfnetlink_conntrack.h>
1931 #include <linux/mutex.h>
1932
1933 /* Generic function for tcp/udp/sctp/dccp and alike. */
1934 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1935                                const struct nf_conntrack_tuple *tuple)
1936 {
1937         if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1938             nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1939                 goto nla_put_failure;
1940         return 0;
1941
1942 nla_put_failure:
1943         return -1;
1944 }
1945 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1946
1947 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1948         [CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
1949         [CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
1950 };
1951 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1952
1953 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1954                                struct nf_conntrack_tuple *t,
1955                                u_int32_t flags)
1956 {
1957         if (flags & CTA_FILTER_FLAG(CTA_PROTO_SRC_PORT)) {
1958                 if (!tb[CTA_PROTO_SRC_PORT])
1959                         return -EINVAL;
1960
1961                 t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1962         }
1963
1964         if (flags & CTA_FILTER_FLAG(CTA_PROTO_DST_PORT)) {
1965                 if (!tb[CTA_PROTO_DST_PORT])
1966                         return -EINVAL;
1967
1968                 t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1969         }
1970
1971         return 0;
1972 }
1973 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1974
1975 unsigned int nf_ct_port_nlattr_tuple_size(void)
1976 {
1977         static unsigned int size __read_mostly;
1978
1979         if (!size)
1980                 size = nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1981
1982         return size;
1983 }
1984 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1985 #endif
1986
1987 /* Used by ipt_REJECT and ip6t_REJECT. */
1988 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb)
1989 {
1990         struct nf_conn *ct;
1991         enum ip_conntrack_info ctinfo;
1992
1993         /* This ICMP is in reverse direction to the packet which caused it */
1994         ct = nf_ct_get(skb, &ctinfo);
1995         if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1996                 ctinfo = IP_CT_RELATED_REPLY;
1997         else
1998                 ctinfo = IP_CT_RELATED;
1999
2000         /* Attach to new skbuff, and increment count */
2001         nf_ct_set(nskb, ct, ctinfo);
2002         nf_conntrack_get(skb_nfct(nskb));
2003 }
2004
2005 static int __nf_conntrack_update(struct net *net, struct sk_buff *skb,
2006                                  struct nf_conn *ct,
2007                                  enum ip_conntrack_info ctinfo)
2008 {
2009         struct nf_conntrack_tuple_hash *h;
2010         struct nf_conntrack_tuple tuple;
2011         struct nf_nat_hook *nat_hook;
2012         unsigned int status;
2013         int dataoff;
2014         u16 l3num;
2015         u8 l4num;
2016
2017         l3num = nf_ct_l3num(ct);
2018
2019         dataoff = get_l4proto(skb, skb_network_offset(skb), l3num, &l4num);
2020         if (dataoff <= 0)
2021                 return -1;
2022
2023         if (!nf_ct_get_tuple(skb, skb_network_offset(skb), dataoff, l3num,
2024                              l4num, net, &tuple))
2025                 return -1;
2026
2027         if (ct->status & IPS_SRC_NAT) {
2028                 memcpy(tuple.src.u3.all,
2029                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.all,
2030                        sizeof(tuple.src.u3.all));
2031                 tuple.src.u.all =
2032                         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u.all;
2033         }
2034
2035         if (ct->status & IPS_DST_NAT) {
2036                 memcpy(tuple.dst.u3.all,
2037                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u3.all,
2038                        sizeof(tuple.dst.u3.all));
2039                 tuple.dst.u.all =
2040                         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u.all;
2041         }
2042
2043         h = nf_conntrack_find_get(net, nf_ct_zone(ct), &tuple);
2044         if (!h)
2045                 return 0;
2046
2047         /* Store status bits of the conntrack that is clashing to re-do NAT
2048          * mangling according to what it has been done already to this packet.
2049          */
2050         status = ct->status;
2051
2052         nf_ct_put(ct);
2053         ct = nf_ct_tuplehash_to_ctrack(h);
2054         nf_ct_set(skb, ct, ctinfo);
2055
2056         nat_hook = rcu_dereference(nf_nat_hook);
2057         if (!nat_hook)
2058                 return 0;
2059
2060         if (status & IPS_SRC_NAT &&
2061             nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_SRC,
2062                                 IP_CT_DIR_ORIGINAL) == NF_DROP)
2063                 return -1;
2064
2065         if (status & IPS_DST_NAT &&
2066             nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_DST,
2067                                 IP_CT_DIR_ORIGINAL) == NF_DROP)
2068                 return -1;
2069
2070         return 0;
2071 }
2072
2073 /* This packet is coming from userspace via nf_queue, complete the packet
2074  * processing after the helper invocation in nf_confirm().
2075  */
2076 static int nf_confirm_cthelper(struct sk_buff *skb, struct nf_conn *ct,
2077                                enum ip_conntrack_info ctinfo)
2078 {
2079         const struct nf_conntrack_helper *helper;
2080         const struct nf_conn_help *help;
2081         int protoff;
2082
2083         help = nfct_help(ct);
2084         if (!help)
2085                 return 0;
2086
2087         helper = rcu_dereference(help->helper);
2088         if (!(helper->flags & NF_CT_HELPER_F_USERSPACE))
2089                 return 0;
2090
2091         switch (nf_ct_l3num(ct)) {
2092         case NFPROTO_IPV4:
2093                 protoff = skb_network_offset(skb) + ip_hdrlen(skb);
2094                 break;
2095 #if IS_ENABLED(CONFIG_IPV6)
2096         case NFPROTO_IPV6: {
2097                 __be16 frag_off;
2098                 u8 pnum;
2099
2100                 pnum = ipv6_hdr(skb)->nexthdr;
2101                 protoff = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &pnum,
2102                                            &frag_off);
2103                 if (protoff < 0 || (frag_off & htons(~0x7)) != 0)
2104                         return 0;
2105                 break;
2106         }
2107 #endif
2108         default:
2109                 return 0;
2110         }
2111
2112         if (test_bit(IPS_SEQ_ADJUST_BIT, &ct->status) &&
2113             !nf_is_loopback_packet(skb)) {
2114                 if (!nf_ct_seq_adjust(skb, ct, ctinfo, protoff)) {
2115                         NF_CT_STAT_INC_ATOMIC(nf_ct_net(ct), drop);
2116                         return -1;
2117                 }
2118         }
2119
2120         /* We've seen it coming out the other side: confirm it */
2121         return nf_conntrack_confirm(skb) == NF_DROP ? - 1 : 0;
2122 }
2123
2124 static int nf_conntrack_update(struct net *net, struct sk_buff *skb)
2125 {
2126         enum ip_conntrack_info ctinfo;
2127         struct nf_conn *ct;
2128         int err;
2129
2130         ct = nf_ct_get(skb, &ctinfo);
2131         if (!ct)
2132                 return 0;
2133
2134         if (!nf_ct_is_confirmed(ct)) {
2135                 err = __nf_conntrack_update(net, skb, ct, ctinfo);
2136                 if (err < 0)
2137                         return err;
2138
2139                 ct = nf_ct_get(skb, &ctinfo);
2140         }
2141
2142         return nf_confirm_cthelper(skb, ct, ctinfo);
2143 }
2144
2145 static bool nf_conntrack_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple,
2146                                        const struct sk_buff *skb)
2147 {
2148         const struct nf_conntrack_tuple *src_tuple;
2149         const struct nf_conntrack_tuple_hash *hash;
2150         struct nf_conntrack_tuple srctuple;
2151         enum ip_conntrack_info ctinfo;
2152         struct nf_conn *ct;
2153
2154         ct = nf_ct_get(skb, &ctinfo);
2155         if (ct) {
2156                 src_tuple = nf_ct_tuple(ct, CTINFO2DIR(ctinfo));
2157                 memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple));
2158                 return true;
2159         }
2160
2161         if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb),
2162                                NFPROTO_IPV4, dev_net(skb->dev),
2163                                &srctuple))
2164                 return false;
2165
2166         hash = nf_conntrack_find_get(dev_net(skb->dev),
2167                                      &nf_ct_zone_dflt,
2168                                      &srctuple);
2169         if (!hash)
2170                 return false;
2171
2172         ct = nf_ct_tuplehash_to_ctrack(hash);
2173         src_tuple = nf_ct_tuple(ct, !hash->tuple.dst.dir);
2174         memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple));
2175         nf_ct_put(ct);
2176
2177         return true;
2178 }
2179
2180 /* Bring out ya dead! */
2181 static struct nf_conn *
2182 get_next_corpse(int (*iter)(struct nf_conn *i, void *data),
2183                 void *data, unsigned int *bucket)
2184 {
2185         struct nf_conntrack_tuple_hash *h;
2186         struct nf_conn *ct;
2187         struct hlist_nulls_node *n;
2188         spinlock_t *lockp;
2189
2190         for (; *bucket < nf_conntrack_htable_size; (*bucket)++) {
2191                 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];
2192                 local_bh_disable();
2193                 nf_conntrack_lock(lockp);
2194                 if (*bucket < nf_conntrack_htable_size) {
2195                         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnnode) {
2196                                 if (NF_CT_DIRECTION(h) != IP_CT_DIR_REPLY)
2197                                         continue;
2198                                 /* All nf_conn objects are added to hash table twice, one
2199                                  * for original direction tuple, once for the reply tuple.
2200                                  *
2201                                  * Exception: In the IPS_NAT_CLASH case, only the reply
2202                                  * tuple is added (the original tuple already existed for
2203                                  * a different object).
2204                                  *
2205                                  * We only need to call the iterator once for each
2206                                  * conntrack, so we just use the 'reply' direction
2207                                  * tuple while iterating.
2208                                  */
2209                                 ct = nf_ct_tuplehash_to_ctrack(h);
2210                                 if (iter(ct, data))
2211                                         goto found;
2212                         }
2213                 }
2214                 spin_unlock(lockp);
2215                 local_bh_enable();
2216                 cond_resched();
2217         }
2218
2219         return NULL;
2220 found:
2221         atomic_inc(&ct->ct_general.use);
2222         spin_unlock(lockp);
2223         local_bh_enable();
2224         return ct;
2225 }
2226
2227 static void nf_ct_iterate_cleanup(int (*iter)(struct nf_conn *i, void *data),
2228                                   void *data, u32 portid, int report)
2229 {
2230         unsigned int bucket = 0, sequence;
2231         struct nf_conn *ct;
2232
2233         might_sleep();
2234
2235         for (;;) {
2236                 sequence = read_seqcount_begin(&nf_conntrack_generation);
2237
2238                 while ((ct = get_next_corpse(iter, data, &bucket)) != NULL) {
2239                         /* Time to push up daises... */
2240
2241                         nf_ct_delete(ct, portid, report);
2242                         nf_ct_put(ct);
2243                         cond_resched();
2244                 }
2245
2246                 if (!read_seqcount_retry(&nf_conntrack_generation, sequence))
2247                         break;
2248                 bucket = 0;
2249         }
2250 }
2251
2252 struct iter_data {
2253         int (*iter)(struct nf_conn *i, void *data);
2254         void *data;
2255         struct net *net;
2256 };
2257
2258 static int iter_net_only(struct nf_conn *i, void *data)
2259 {
2260         struct iter_data *d = data;
2261
2262         if (!net_eq(d->net, nf_ct_net(i)))
2263                 return 0;
2264
2265         return d->iter(i, d->data);
2266 }
2267
2268 static void
2269 __nf_ct_unconfirmed_destroy(struct net *net)
2270 {
2271         int cpu;
2272
2273         for_each_possible_cpu(cpu) {
2274                 struct nf_conntrack_tuple_hash *h;
2275                 struct hlist_nulls_node *n;
2276                 struct ct_pcpu *pcpu;
2277
2278                 pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
2279
2280                 spin_lock_bh(&pcpu->lock);
2281                 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) {
2282                         struct nf_conn *ct;
2283
2284                         ct = nf_ct_tuplehash_to_ctrack(h);
2285
2286                         /* we cannot call iter() on unconfirmed list, the
2287                          * owning cpu can reallocate ct->ext at any time.
2288                          */
2289                         set_bit(IPS_DYING_BIT, &ct->status);
2290                 }
2291                 spin_unlock_bh(&pcpu->lock);
2292                 cond_resched();
2293         }
2294 }
2295
2296 void nf_ct_unconfirmed_destroy(struct net *net)
2297 {
2298         struct nf_conntrack_net *cnet = net_generic(net, nf_conntrack_net_id);
2299
2300         might_sleep();
2301
2302         if (atomic_read(&cnet->count) > 0) {
2303                 __nf_ct_unconfirmed_destroy(net);
2304                 nf_queue_nf_hook_drop(net);
2305                 synchronize_net();
2306         }
2307 }
2308 EXPORT_SYMBOL_GPL(nf_ct_unconfirmed_destroy);
2309
2310 void nf_ct_iterate_cleanup_net(struct net *net,
2311                                int (*iter)(struct nf_conn *i, void *data),
2312                                void *data, u32 portid, int report)
2313 {
2314         struct nf_conntrack_net *cnet = net_generic(net, nf_conntrack_net_id);
2315         struct iter_data d;
2316
2317         might_sleep();
2318
2319         if (atomic_read(&cnet->count) == 0)
2320                 return;
2321
2322         d.iter = iter;
2323         d.data = data;
2324         d.net = net;
2325
2326         nf_ct_iterate_cleanup(iter_net_only, &d, portid, report);
2327 }
2328 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup_net);
2329
2330 /**
2331  * nf_ct_iterate_destroy - destroy unconfirmed conntracks and iterate table
2332  * @iter: callback to invoke for each conntrack
2333  * @data: data to pass to @iter
2334  *
2335  * Like nf_ct_iterate_cleanup, but first marks conntracks on the
2336  * unconfirmed list as dying (so they will not be inserted into
2337  * main table).
2338  *
2339  * Can only be called in module exit path.
2340  */
2341 void
2342 nf_ct_iterate_destroy(int (*iter)(struct nf_conn *i, void *data), void *data)
2343 {
2344         struct net *net;
2345
2346         down_read(&net_rwsem);
2347         for_each_net(net) {
2348                 struct nf_conntrack_net *cnet = net_generic(net, nf_conntrack_net_id);
2349
2350                 if (atomic_read(&cnet->count) == 0)
2351                         continue;
2352                 __nf_ct_unconfirmed_destroy(net);
2353                 nf_queue_nf_hook_drop(net);
2354         }
2355         up_read(&net_rwsem);
2356
2357         /* Need to wait for netns cleanup worker to finish, if its
2358          * running -- it might have deleted a net namespace from
2359          * the global list, so our __nf_ct_unconfirmed_destroy() might
2360          * not have affected all namespaces.
2361          */
2362         net_ns_barrier();
2363
2364         /* a conntrack could have been unlinked from unconfirmed list
2365          * before we grabbed pcpu lock in __nf_ct_unconfirmed_destroy().
2366          * This makes sure its inserted into conntrack table.
2367          */
2368         synchronize_net();
2369
2370         nf_ct_iterate_cleanup(iter, data, 0, 0);
2371 }
2372 EXPORT_SYMBOL_GPL(nf_ct_iterate_destroy);
2373
2374 static int kill_all(struct nf_conn *i, void *data)
2375 {
2376         return net_eq(nf_ct_net(i), data);
2377 }
2378
2379 void nf_conntrack_cleanup_start(void)
2380 {
2381         conntrack_gc_work.exiting = true;
2382         RCU_INIT_POINTER(ip_ct_attach, NULL);
2383 }
2384
2385 void nf_conntrack_cleanup_end(void)
2386 {
2387         RCU_INIT_POINTER(nf_ct_hook, NULL);
2388         cancel_delayed_work_sync(&conntrack_gc_work.dwork);
2389         kvfree(nf_conntrack_hash);
2390
2391         nf_conntrack_proto_fini();
2392         nf_conntrack_seqadj_fini();
2393         nf_conntrack_labels_fini();
2394         nf_conntrack_helper_fini();
2395         nf_conntrack_timeout_fini();
2396         nf_conntrack_ecache_fini();
2397         nf_conntrack_tstamp_fini();
2398         nf_conntrack_acct_fini();
2399         nf_conntrack_expect_fini();
2400
2401         kmem_cache_destroy(nf_conntrack_cachep);
2402 }
2403
2404 /*
2405  * Mishearing the voices in his head, our hero wonders how he's
2406  * supposed to kill the mall.
2407  */
2408 void nf_conntrack_cleanup_net(struct net *net)
2409 {
2410         LIST_HEAD(single);
2411
2412         list_add(&net->exit_list, &single);
2413         nf_conntrack_cleanup_net_list(&single);
2414 }
2415
2416 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
2417 {
2418         int busy;
2419         struct net *net;
2420
2421         /*
2422          * This makes sure all current packets have passed through
2423          *  netfilter framework.  Roll on, two-stage module
2424          *  delete...
2425          */
2426         synchronize_net();
2427 i_see_dead_people:
2428         busy = 0;
2429         list_for_each_entry(net, net_exit_list, exit_list) {
2430                 struct nf_conntrack_net *cnet = net_generic(net, nf_conntrack_net_id);
2431
2432                 nf_ct_iterate_cleanup(kill_all, net, 0, 0);
2433                 if (atomic_read(&cnet->count) != 0)
2434                         busy = 1;
2435         }
2436         if (busy) {
2437                 schedule();
2438                 goto i_see_dead_people;
2439         }
2440
2441         list_for_each_entry(net, net_exit_list, exit_list) {
2442                 nf_conntrack_proto_pernet_fini(net);
2443                 nf_conntrack_ecache_pernet_fini(net);
2444                 nf_conntrack_expect_pernet_fini(net);
2445                 free_percpu(net->ct.stat);
2446                 free_percpu(net->ct.pcpu_lists);
2447         }
2448 }
2449
2450 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
2451 {
2452         struct hlist_nulls_head *hash;
2453         unsigned int nr_slots, i;
2454
2455         if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head)))
2456                 return NULL;
2457
2458         BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
2459         nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
2460
2461         hash = kvcalloc(nr_slots, sizeof(struct hlist_nulls_head), GFP_KERNEL);
2462
2463         if (hash && nulls)
2464                 for (i = 0; i < nr_slots; i++)
2465                         INIT_HLIST_NULLS_HEAD(&hash[i], i);
2466
2467         return hash;
2468 }
2469 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
2470
2471 int nf_conntrack_hash_resize(unsigned int hashsize)
2472 {
2473         int i, bucket;
2474         unsigned int old_size;
2475         struct hlist_nulls_head *hash, *old_hash;
2476         struct nf_conntrack_tuple_hash *h;
2477         struct nf_conn *ct;
2478
2479         if (!hashsize)
2480                 return -EINVAL;
2481
2482         hash = nf_ct_alloc_hashtable(&hashsize, 1);
2483         if (!hash)
2484                 return -ENOMEM;
2485
2486         old_size = nf_conntrack_htable_size;
2487         if (old_size == hashsize) {
2488                 kvfree(hash);
2489                 return 0;
2490         }
2491
2492         local_bh_disable();
2493         nf_conntrack_all_lock();
2494         write_seqcount_begin(&nf_conntrack_generation);
2495
2496         /* Lookups in the old hash might happen in parallel, which means we
2497          * might get false negatives during connection lookup. New connections
2498          * created because of a false negative won't make it into the hash
2499          * though since that required taking the locks.
2500          */
2501
2502         for (i = 0; i < nf_conntrack_htable_size; i++) {
2503                 while (!hlist_nulls_empty(&nf_conntrack_hash[i])) {
2504                         h = hlist_nulls_entry(nf_conntrack_hash[i].first,
2505                                               struct nf_conntrack_tuple_hash, hnnode);
2506                         ct = nf_ct_tuplehash_to_ctrack(h);
2507                         hlist_nulls_del_rcu(&h->hnnode);
2508                         bucket = __hash_conntrack(nf_ct_net(ct),
2509                                                   &h->tuple, hashsize);
2510                         hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
2511                 }
2512         }
2513         old_size = nf_conntrack_htable_size;
2514         old_hash = nf_conntrack_hash;
2515
2516         nf_conntrack_hash = hash;
2517         nf_conntrack_htable_size = hashsize;
2518
2519         write_seqcount_end(&nf_conntrack_generation);
2520         nf_conntrack_all_unlock();
2521         local_bh_enable();
2522
2523         synchronize_net();
2524         kvfree(old_hash);
2525         return 0;
2526 }
2527
2528 int nf_conntrack_set_hashsize(const char *val, const struct kernel_param *kp)
2529 {
2530         unsigned int hashsize;
2531         int rc;
2532
2533         if (current->nsproxy->net_ns != &init_net)
2534                 return -EOPNOTSUPP;
2535
2536         /* On boot, we can set this without any fancy locking. */
2537         if (!nf_conntrack_hash)
2538                 return param_set_uint(val, kp);
2539
2540         rc = kstrtouint(val, 0, &hashsize);
2541         if (rc)
2542                 return rc;
2543
2544         return nf_conntrack_hash_resize(hashsize);
2545 }
2546
2547 static __always_inline unsigned int total_extension_size(void)
2548 {
2549         /* remember to add new extensions below */
2550         BUILD_BUG_ON(NF_CT_EXT_NUM > 9);
2551
2552         return sizeof(struct nf_ct_ext) +
2553                sizeof(struct nf_conn_help)
2554 #if IS_ENABLED(CONFIG_NF_NAT)
2555                 + sizeof(struct nf_conn_nat)
2556 #endif
2557                 + sizeof(struct nf_conn_seqadj)
2558                 + sizeof(struct nf_conn_acct)
2559 #ifdef CONFIG_NF_CONNTRACK_EVENTS
2560                 + sizeof(struct nf_conntrack_ecache)
2561 #endif
2562 #ifdef CONFIG_NF_CONNTRACK_TIMESTAMP
2563                 + sizeof(struct nf_conn_tstamp)
2564 #endif
2565 #ifdef CONFIG_NF_CONNTRACK_TIMEOUT
2566                 + sizeof(struct nf_conn_timeout)
2567 #endif
2568 #ifdef CONFIG_NF_CONNTRACK_LABELS
2569                 + sizeof(struct nf_conn_labels)
2570 #endif
2571 #if IS_ENABLED(CONFIG_NETFILTER_SYNPROXY)
2572                 + sizeof(struct nf_conn_synproxy)
2573 #endif
2574         ;
2575 };
2576
2577 int nf_conntrack_init_start(void)
2578 {
2579         unsigned long nr_pages = totalram_pages();
2580         int max_factor = 8;
2581         int ret = -ENOMEM;
2582         int i;
2583
2584         /* struct nf_ct_ext uses u8 to store offsets/size */
2585         BUILD_BUG_ON(total_extension_size() > 255u);
2586
2587         seqcount_spinlock_init(&nf_conntrack_generation,
2588                                &nf_conntrack_locks_all_lock);
2589
2590         for (i = 0; i < CONNTRACK_LOCKS; i++)
2591                 spin_lock_init(&nf_conntrack_locks[i]);
2592
2593         if (!nf_conntrack_htable_size) {
2594                 /* Idea from tcp.c: use 1/16384 of memory.
2595                  * On i386: 32MB machine has 512 buckets.
2596                  * >= 1GB machines have 16384 buckets.
2597                  * >= 4GB machines have 65536 buckets.
2598                  */
2599                 nf_conntrack_htable_size
2600                         = (((nr_pages << PAGE_SHIFT) / 16384)
2601                            / sizeof(struct hlist_head));
2602                 if (nr_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))
2603                         nf_conntrack_htable_size = 65536;
2604                 else if (nr_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
2605                         nf_conntrack_htable_size = 16384;
2606                 if (nf_conntrack_htable_size < 32)
2607                         nf_conntrack_htable_size = 32;
2608
2609                 /* Use a max. factor of four by default to get the same max as
2610                  * with the old struct list_heads. When a table size is given
2611                  * we use the old value of 8 to avoid reducing the max.
2612                  * entries. */
2613                 max_factor = 4;
2614         }
2615
2616         nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1);
2617         if (!nf_conntrack_hash)
2618                 return -ENOMEM;
2619
2620         nf_conntrack_max = max_factor * nf_conntrack_htable_size;
2621
2622         nf_conntrack_cachep = kmem_cache_create("nf_conntrack",
2623                                                 sizeof(struct nf_conn),
2624                                                 NFCT_INFOMASK + 1,
2625                                                 SLAB_TYPESAFE_BY_RCU | SLAB_HWCACHE_ALIGN, NULL);
2626         if (!nf_conntrack_cachep)
2627                 goto err_cachep;
2628
2629         ret = nf_conntrack_expect_init();
2630         if (ret < 0)
2631                 goto err_expect;
2632
2633         ret = nf_conntrack_acct_init();
2634         if (ret < 0)
2635                 goto err_acct;
2636
2637         ret = nf_conntrack_tstamp_init();
2638         if (ret < 0)
2639                 goto err_tstamp;
2640
2641         ret = nf_conntrack_ecache_init();
2642         if (ret < 0)
2643                 goto err_ecache;
2644
2645         ret = nf_conntrack_timeout_init();
2646         if (ret < 0)
2647                 goto err_timeout;
2648
2649         ret = nf_conntrack_helper_init();
2650         if (ret < 0)
2651                 goto err_helper;
2652
2653         ret = nf_conntrack_labels_init();
2654         if (ret < 0)
2655                 goto err_labels;
2656
2657         ret = nf_conntrack_seqadj_init();
2658         if (ret < 0)
2659                 goto err_seqadj;
2660
2661         ret = nf_conntrack_proto_init();
2662         if (ret < 0)
2663                 goto err_proto;
2664
2665         conntrack_gc_work_init(&conntrack_gc_work);
2666         queue_delayed_work(system_power_efficient_wq, &conntrack_gc_work.dwork, HZ);
2667
2668         return 0;
2669
2670 err_proto:
2671         nf_conntrack_seqadj_fini();
2672 err_seqadj:
2673         nf_conntrack_labels_fini();
2674 err_labels:
2675         nf_conntrack_helper_fini();
2676 err_helper:
2677         nf_conntrack_timeout_fini();
2678 err_timeout:
2679         nf_conntrack_ecache_fini();
2680 err_ecache:
2681         nf_conntrack_tstamp_fini();
2682 err_tstamp:
2683         nf_conntrack_acct_fini();
2684 err_acct:
2685         nf_conntrack_expect_fini();
2686 err_expect:
2687         kmem_cache_destroy(nf_conntrack_cachep);
2688 err_cachep:
2689         kvfree(nf_conntrack_hash);
2690         return ret;
2691 }
2692
2693 static struct nf_ct_hook nf_conntrack_hook = {
2694         .update         = nf_conntrack_update,
2695         .destroy        = destroy_conntrack,
2696         .get_tuple_skb  = nf_conntrack_get_tuple_skb,
2697 };
2698
2699 void nf_conntrack_init_end(void)
2700 {
2701         /* For use by REJECT target */
2702         RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
2703         RCU_INIT_POINTER(nf_ct_hook, &nf_conntrack_hook);
2704 }
2705
2706 /*
2707  * We need to use special "null" values, not used in hash table
2708  */
2709 #define UNCONFIRMED_NULLS_VAL   ((1<<30)+0)
2710 #define DYING_NULLS_VAL         ((1<<30)+1)
2711
2712 int nf_conntrack_init_net(struct net *net)
2713 {
2714         struct nf_conntrack_net *cnet = net_generic(net, nf_conntrack_net_id);
2715         int ret = -ENOMEM;
2716         int cpu;
2717
2718         BUILD_BUG_ON(IP_CT_UNTRACKED == IP_CT_NUMBER);
2719         BUILD_BUG_ON_NOT_POWER_OF_2(CONNTRACK_LOCKS);
2720         atomic_set(&cnet->count, 0);
2721
2722         net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu);
2723         if (!net->ct.pcpu_lists)
2724                 goto err_stat;
2725
2726         for_each_possible_cpu(cpu) {
2727                 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
2728
2729                 spin_lock_init(&pcpu->lock);
2730                 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL);
2731                 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL);
2732         }
2733
2734         net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
2735         if (!net->ct.stat)
2736                 goto err_pcpu_lists;
2737
2738         ret = nf_conntrack_expect_pernet_init(net);
2739         if (ret < 0)
2740                 goto err_expect;
2741
2742         nf_conntrack_acct_pernet_init(net);
2743         nf_conntrack_tstamp_pernet_init(net);
2744         nf_conntrack_ecache_pernet_init(net);
2745         nf_conntrack_helper_pernet_init(net);
2746         nf_conntrack_proto_pernet_init(net);
2747
2748         return 0;
2749
2750 err_expect:
2751         free_percpu(net->ct.stat);
2752 err_pcpu_lists:
2753         free_percpu(net->ct.pcpu_lists);
2754 err_stat:
2755         return ret;
2756 }