net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27
  28 unsigned int nf_tables_net_id __read_mostly;
  29
  30 static LIST_HEAD(nf_tables_expressions);
  31 static LIST_HEAD(nf_tables_objects);
  32 static LIST_HEAD(nf_tables_flowtables);
  33 static LIST_HEAD(nf_tables_destroy_list);
  34 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  35
  36 enum {
  37         NFT_VALIDATE_SKIP       = 0,
  38         NFT_VALIDATE_NEED,
  39         NFT_VALIDATE_DO,
  40 };
  41
  42 static struct rhltable nft_objname_ht;
  43
  44 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  45 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  46 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  47
  48 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  49 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  50 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  51
  52 static const struct rhashtable_params nft_chain_ht_params = {
  53         .head_offset            = offsetof(struct nft_chain, rhlhead),
  54         .key_offset             = offsetof(struct nft_chain, name),
  55         .hashfn                 = nft_chain_hash,
  56         .obj_hashfn             = nft_chain_hash_obj,
  57         .obj_cmpfn              = nft_chain_hash_cmp,
  58         .automatic_shrinking    = true,
  59 };
  60
  61 static const struct rhashtable_params nft_objname_ht_params = {
  62         .head_offset            = offsetof(struct nft_object, rhlhead),
  63         .key_offset             = offsetof(struct nft_object, key),
  64         .hashfn                 = nft_objname_hash,
  65         .obj_hashfn             = nft_objname_hash_obj,
  66         .obj_cmpfn              = nft_objname_hash_cmp,
  67         .automatic_shrinking    = true,
  68 };
  69
  70 struct nft_audit_data {
  71         struct nft_table *table;
  72         int entries;
  73         int op;
  74         struct list_head list;
  75 };
  76
  77 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  78         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  79         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  80         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  81         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  82         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  83         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  84         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  85         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  87         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  88         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  90         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  91         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  93         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  94         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  96         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
  97         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
  99         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 100         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 101         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 102         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 103 };
 104
 105 static void nft_validate_state_update(struct net *net, u8 new_validate_state)
 106 {
 107         struct nftables_pernet *nft_net = nft_pernet(net);
 108
 109         switch (nft_net->validate_state) {
 110         case NFT_VALIDATE_SKIP:
 111                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 112                 break;
 113         case NFT_VALIDATE_NEED:
 114                 break;
 115         case NFT_VALIDATE_DO:
 116                 if (new_validate_state == NFT_VALIDATE_NEED)
 117                         return;
 118         }
 119
 120         nft_net->validate_state = new_validate_state;
 121 }
 122 static void nf_tables_trans_destroy_work(struct work_struct *w);
 123 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 124
 125 static void nft_ctx_init(struct nft_ctx *ctx,
 126                          struct net *net,
 127                          const struct sk_buff *skb,
 128                          const struct nlmsghdr *nlh,
 129                          u8 family,
 130                          struct nft_table *table,
 131                          struct nft_chain *chain,
 132                          const struct nlattr * const *nla)
 133 {
 134         ctx->net        = net;
 135         ctx->family     = family;
 136         ctx->level      = 0;
 137         ctx->table      = table;
 138         ctx->chain      = chain;
 139         ctx->nla        = nla;
 140         ctx->portid     = NETLINK_CB(skb).portid;
 141         ctx->report     = nlmsg_report(nlh);
 142         ctx->flags      = nlh->nlmsg_flags;
 143         ctx->seq        = nlh->nlmsg_seq;
 144 }
 145
 146 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 147                                              int msg_type, u32 size, gfp_t gfp)
 148 {
 149         struct nft_trans *trans;
 150
 151         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 152         if (trans == NULL)
 153                 return NULL;
 154
 155         INIT_LIST_HEAD(&trans->list);
 156         trans->msg_type = msg_type;
 157         trans->ctx      = *ctx;
 158
 159         return trans;
 160 }
 161
 162 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 163                                          int msg_type, u32 size)
 164 {
 165         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 166 }
 167
 168 static void nft_trans_destroy(struct nft_trans *trans)
 169 {
 170         list_del(&trans->list);
 171         kfree(trans);
 172 }
 173
 174 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 175 {
 176         struct nftables_pernet *nft_net;
 177         struct net *net = ctx->net;
 178         struct nft_trans *trans;
 179
 180         if (!nft_set_is_anonymous(set))
 181                 return;
 182
 183         nft_net = nft_pernet(net);
 184         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 185                 switch (trans->msg_type) {
 186                 case NFT_MSG_NEWSET:
 187                         if (nft_trans_set(trans) == set)
 188                                 nft_trans_set_bound(trans) = true;
 189                         break;
 190                 case NFT_MSG_NEWSETELEM:
 191                         if (nft_trans_elem_set(trans) == set)
 192                                 nft_trans_elem_set_bound(trans) = true;
 193                         break;
 194                 }
 195         }
 196 }
 197
 198 static int nft_netdev_register_hooks(struct net *net,
 199                                      struct list_head *hook_list)
 200 {
 201         struct nft_hook *hook;
 202         int err, j;
 203
 204         j = 0;
 205         list_for_each_entry(hook, hook_list, list) {
 206                 err = nf_register_net_hook(net, &hook->ops);
 207                 if (err < 0)
 208                         goto err_register;
 209
 210                 j++;
 211         }
 212         return 0;
 213
 214 err_register:
 215         list_for_each_entry(hook, hook_list, list) {
 216                 if (j-- <= 0)
 217                         break;
 218
 219                 nf_unregister_net_hook(net, &hook->ops);
 220         }
 221         return err;
 222 }
 223
 224 static void nft_netdev_unregister_hooks(struct net *net,
 225                                         struct list_head *hook_list,
 226                                         bool release_netdev)
 227 {
 228         struct nft_hook *hook, *next;
 229
 230         list_for_each_entry_safe(hook, next, hook_list, list) {
 231                 nf_unregister_net_hook(net, &hook->ops);
 232                 if (release_netdev) {
 233                         list_del(&hook->list);
 234                         kfree_rcu(hook, rcu);
 235                 }
 236         }
 237 }
 238
 239 static int nf_tables_register_hook(struct net *net,
 240                                    const struct nft_table *table,
 241                                    struct nft_chain *chain)
 242 {
 243         struct nft_base_chain *basechain;
 244         const struct nf_hook_ops *ops;
 245
 246         if (table->flags & NFT_TABLE_F_DORMANT ||
 247             !nft_is_base_chain(chain))
 248                 return 0;
 249
 250         basechain = nft_base_chain(chain);
 251         ops = &basechain->ops;
 252
 253         if (basechain->type->ops_register)
 254                 return basechain->type->ops_register(net, ops);
 255
 256         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 257                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 258
 259         return nf_register_net_hook(net, &basechain->ops);
 260 }
 261
 262 static void __nf_tables_unregister_hook(struct net *net,
 263                                         const struct nft_table *table,
 264                                         struct nft_chain *chain,
 265                                         bool release_netdev)
 266 {
 267         struct nft_base_chain *basechain;
 268         const struct nf_hook_ops *ops;
 269
 270         if (table->flags & NFT_TABLE_F_DORMANT ||
 271             !nft_is_base_chain(chain))
 272                 return;
 273         basechain = nft_base_chain(chain);
 274         ops = &basechain->ops;
 275
 276         if (basechain->type->ops_unregister)
 277                 return basechain->type->ops_unregister(net, ops);
 278
 279         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 280                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 281                                             release_netdev);
 282         else
 283                 nf_unregister_net_hook(net, &basechain->ops);
 284 }
 285
 286 static void nf_tables_unregister_hook(struct net *net,
 287                                       const struct nft_table *table,
 288                                       struct nft_chain *chain)
 289 {
 290         return __nf_tables_unregister_hook(net, table, chain, false);
 291 }
 292
 293 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 294 {
 295         struct nftables_pernet *nft_net = nft_pernet(net);
 296
 297         list_add_tail(&trans->list, &nft_net->commit_list);
 298 }
 299
 300 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 301 {
 302         struct nft_trans *trans;
 303
 304         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 305         if (trans == NULL)
 306                 return -ENOMEM;
 307
 308         if (msg_type == NFT_MSG_NEWTABLE)
 309                 nft_activate_next(ctx->net, ctx->table);
 310
 311         nft_trans_commit_list_add_tail(ctx->net, trans);
 312         return 0;
 313 }
 314
 315 static int nft_deltable(struct nft_ctx *ctx)
 316 {
 317         int err;
 318
 319         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 320         if (err < 0)
 321                 return err;
 322
 323         nft_deactivate_next(ctx->net, ctx->table);
 324         return err;
 325 }
 326
 327 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 328 {
 329         struct nft_trans *trans;
 330
 331         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 332         if (trans == NULL)
 333                 return ERR_PTR(-ENOMEM);
 334
 335         if (msg_type == NFT_MSG_NEWCHAIN) {
 336                 nft_activate_next(ctx->net, ctx->chain);
 337
 338                 if (ctx->nla[NFTA_CHAIN_ID]) {
 339                         nft_trans_chain_id(trans) =
 340                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 341                 }
 342         }
 343
 344         nft_trans_commit_list_add_tail(ctx->net, trans);
 345         return trans;
 346 }
 347
 348 static int nft_delchain(struct nft_ctx *ctx)
 349 {
 350         struct nft_trans *trans;
 351
 352         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 353         if (IS_ERR(trans))
 354                 return PTR_ERR(trans);
 355
 356         ctx->table->use--;
 357         nft_deactivate_next(ctx->net, ctx->chain);
 358
 359         return 0;
 360 }
 361
 362 static void nft_rule_expr_activate(const struct nft_ctx *ctx,
 363                                    struct nft_rule *rule)
 364 {
 365         struct nft_expr *expr;
 366
 367         expr = nft_expr_first(rule);
 368         while (nft_expr_more(rule, expr)) {
 369                 if (expr->ops->activate)
 370                         expr->ops->activate(ctx, expr);
 371
 372                 expr = nft_expr_next(expr);
 373         }
 374 }
 375
 376 static void nft_rule_expr_deactivate(const struct nft_ctx *ctx,
 377                                      struct nft_rule *rule,
 378                                      enum nft_trans_phase phase)
 379 {
 380         struct nft_expr *expr;
 381
 382         expr = nft_expr_first(rule);
 383         while (nft_expr_more(rule, expr)) {
 384                 if (expr->ops->deactivate)
 385                         expr->ops->deactivate(ctx, expr, phase);
 386
 387                 expr = nft_expr_next(expr);
 388         }
 389 }
 390
 391 static int
 392 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 393 {
 394         /* You cannot delete the same rule twice */
 395         if (nft_is_active_next(ctx->net, rule)) {
 396                 nft_deactivate_next(ctx->net, rule);
 397                 ctx->chain->use--;
 398                 return 0;
 399         }
 400         return -ENOENT;
 401 }
 402
 403 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 404                                             struct nft_rule *rule)
 405 {
 406         struct nft_trans *trans;
 407
 408         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 409         if (trans == NULL)
 410                 return NULL;
 411
 412         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 413                 nft_trans_rule_id(trans) =
 414                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 415         }
 416         nft_trans_rule(trans) = rule;
 417         nft_trans_commit_list_add_tail(ctx->net, trans);
 418
 419         return trans;
 420 }
 421
 422 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 423 {
 424         struct nft_flow_rule *flow;
 425         struct nft_trans *trans;
 426         int err;
 427
 428         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 429         if (trans == NULL)
 430                 return -ENOMEM;
 431
 432         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 433                 flow = nft_flow_rule_create(ctx->net, rule);
 434                 if (IS_ERR(flow)) {
 435                         nft_trans_destroy(trans);
 436                         return PTR_ERR(flow);
 437                 }
 438
 439                 nft_trans_flow_rule(trans) = flow;
 440         }
 441
 442         err = nf_tables_delrule_deactivate(ctx, rule);
 443         if (err < 0) {
 444                 nft_trans_destroy(trans);
 445                 return err;
 446         }
 447         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 448
 449         return 0;
 450 }
 451
 452 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 453 {
 454         struct nft_rule *rule;
 455         int err;
 456
 457         list_for_each_entry(rule, &ctx->chain->rules, list) {
 458                 if (!nft_is_active_next(ctx->net, rule))
 459                         continue;
 460
 461                 err = nft_delrule(ctx, rule);
 462                 if (err < 0)
 463                         return err;
 464         }
 465         return 0;
 466 }
 467
 468 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 469                                struct nft_set *set,
 470                                const struct nft_set_desc *desc)
 471 {
 472         struct nft_trans *trans;
 473
 474         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 475         if (trans == NULL)
 476                 return -ENOMEM;
 477
 478         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 479                 nft_trans_set_id(trans) =
 480                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 481                 nft_activate_next(ctx->net, set);
 482         }
 483         nft_trans_set(trans) = set;
 484         if (desc) {
 485                 nft_trans_set_update(trans) = true;
 486                 nft_trans_set_gc_int(trans) = desc->gc_int;
 487                 nft_trans_set_timeout(trans) = desc->timeout;
 488         }
 489         nft_trans_commit_list_add_tail(ctx->net, trans);
 490
 491         return 0;
 492 }
 493
 494 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 495                              struct nft_set *set)
 496 {
 497         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 498 }
 499
 500 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 501 {
 502         int err;
 503
 504         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 505         if (err < 0)
 506                 return err;
 507
 508         nft_deactivate_next(ctx->net, set);
 509         ctx->table->use--;
 510
 511         return err;
 512 }
 513
 514 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 515                              struct nft_object *obj)
 516 {
 517         struct nft_trans *trans;
 518
 519         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 520         if (trans == NULL)
 521                 return -ENOMEM;
 522
 523         if (msg_type == NFT_MSG_NEWOBJ)
 524                 nft_activate_next(ctx->net, obj);
 525
 526         nft_trans_obj(trans) = obj;
 527         nft_trans_commit_list_add_tail(ctx->net, trans);
 528
 529         return 0;
 530 }
 531
 532 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 533 {
 534         int err;
 535
 536         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 537         if (err < 0)
 538                 return err;
 539
 540         nft_deactivate_next(ctx->net, obj);
 541         ctx->table->use--;
 542
 543         return err;
 544 }
 545
 546 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 547                                    struct nft_flowtable *flowtable)
 548 {
 549         struct nft_trans *trans;
 550
 551         trans = nft_trans_alloc(ctx, msg_type,
 552                                 sizeof(struct nft_trans_flowtable));
 553         if (trans == NULL)
 554                 return -ENOMEM;
 555
 556         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 557                 nft_activate_next(ctx->net, flowtable);
 558
 559         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 560         nft_trans_flowtable(trans) = flowtable;
 561         nft_trans_commit_list_add_tail(ctx->net, trans);
 562
 563         return 0;
 564 }
 565
 566 static int nft_delflowtable(struct nft_ctx *ctx,
 567                             struct nft_flowtable *flowtable)
 568 {
 569         int err;
 570
 571         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 572         if (err < 0)
 573                 return err;
 574
 575         nft_deactivate_next(ctx->net, flowtable);
 576         ctx->table->use--;
 577
 578         return err;
 579 }
 580
 581 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 582 {
 583         int i;
 584
 585         for (i = track->regs[dreg].num_reg; i > 0; i--)
 586                 __nft_reg_track_cancel(track, dreg - i);
 587 }
 588
 589 static void __nft_reg_track_update(struct nft_regs_track *track,
 590                                    const struct nft_expr *expr,
 591                                    u8 dreg, u8 num_reg)
 592 {
 593         track->regs[dreg].selector = expr;
 594         track->regs[dreg].bitwise = NULL;
 595         track->regs[dreg].num_reg = num_reg;
 596 }
 597
 598 void nft_reg_track_update(struct nft_regs_track *track,
 599                           const struct nft_expr *expr, u8 dreg, u8 len)
 600 {
 601         unsigned int regcount;
 602         int i;
 603
 604         __nft_reg_track_clobber(track, dreg);
 605
 606         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 607         for (i = 0; i < regcount; i++, dreg++)
 608                 __nft_reg_track_update(track, expr, dreg, i);
 609 }
 610 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 611
 612 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 613 {
 614         unsigned int regcount;
 615         int i;
 616
 617         __nft_reg_track_clobber(track, dreg);
 618
 619         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 620         for (i = 0; i < regcount; i++, dreg++)
 621                 __nft_reg_track_cancel(track, dreg);
 622 }
 623 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 624
 625 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 626 {
 627         track->regs[dreg].selector = NULL;
 628         track->regs[dreg].bitwise = NULL;
 629         track->regs[dreg].num_reg = 0;
 630 }
 631 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 632
 633 /*
 634  * Tables
 635  */
 636
 637 static struct nft_table *nft_table_lookup(const struct net *net,
 638                                           const struct nlattr *nla,
 639                                           u8 family, u8 genmask, u32 nlpid)
 640 {
 641         struct nftables_pernet *nft_net;
 642         struct nft_table *table;
 643
 644         if (nla == NULL)
 645                 return ERR_PTR(-EINVAL);
 646
 647         nft_net = nft_pernet(net);
 648         list_for_each_entry_rcu(table, &nft_net->tables, list,
 649                                 lockdep_is_held(&nft_net->commit_mutex)) {
 650                 if (!nla_strcmp(nla, table->name) &&
 651                     table->family == family &&
 652                     nft_active_genmask(table, genmask)) {
 653                         if (nft_table_has_owner(table) &&
 654                             nlpid && table->nlpid != nlpid)
 655                                 return ERR_PTR(-EPERM);
 656
 657                         return table;
 658                 }
 659         }
 660
 661         return ERR_PTR(-ENOENT);
 662 }
 663
 664 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 665                                                    const struct nlattr *nla,
 666                                                    u8 genmask, u32 nlpid)
 667 {
 668         struct nftables_pernet *nft_net;
 669         struct nft_table *table;
 670
 671         nft_net = nft_pernet(net);
 672         list_for_each_entry(table, &nft_net->tables, list) {
 673                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 674                     nft_active_genmask(table, genmask)) {
 675                         if (nft_table_has_owner(table) &&
 676                             nlpid && table->nlpid != nlpid)
 677                                 return ERR_PTR(-EPERM);
 678
 679                         return table;
 680                 }
 681         }
 682
 683         return ERR_PTR(-ENOENT);
 684 }
 685
 686 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 687 {
 688         return ++table->hgenerator;
 689 }
 690
 691 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 692
 693 static const struct nft_chain_type *
 694 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 695 {
 696         if (family >= NFPROTO_NUMPROTO ||
 697             type >= NFT_CHAIN_T_MAX)
 698                 return NULL;
 699
 700         return chain_type[family][type];
 701 }
 702
 703 static const struct nft_chain_type *
 704 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 705 {
 706         const struct nft_chain_type *type;
 707         int i;
 708
 709         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 710                 type = __nft_chain_type_get(family, i);
 711                 if (!type)
 712                         continue;
 713                 if (!nla_strcmp(nla, type->name))
 714                         return type;
 715         }
 716         return NULL;
 717 }
 718
 719 struct nft_module_request {
 720         struct list_head        list;
 721         char                    module[MODULE_NAME_LEN];
 722         bool                    done;
 723 };
 724
 725 #ifdef CONFIG_MODULES
 726 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 727                                       ...)
 728 {
 729         char module_name[MODULE_NAME_LEN];
 730         struct nftables_pernet *nft_net;
 731         struct nft_module_request *req;
 732         va_list args;
 733         int ret;
 734
 735         va_start(args, fmt);
 736         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 737         va_end(args);
 738         if (ret >= MODULE_NAME_LEN)
 739                 return 0;
 740
 741         nft_net = nft_pernet(net);
 742         list_for_each_entry(req, &nft_net->module_list, list) {
 743                 if (!strcmp(req->module, module_name)) {
 744                         if (req->done)
 745                                 return 0;
 746
 747                         /* A request to load this module already exists. */
 748                         return -EAGAIN;
 749                 }
 750         }
 751
 752         req = kmalloc(sizeof(*req), GFP_KERNEL);
 753         if (!req)
 754                 return -ENOMEM;
 755
 756         req->done = false;
 757         strscpy(req->module, module_name, MODULE_NAME_LEN);
 758         list_add_tail(&req->list, &nft_net->module_list);
 759
 760         return -EAGAIN;
 761 }
 762 EXPORT_SYMBOL_GPL(nft_request_module);
 763 #endif
 764
 765 static void lockdep_nfnl_nft_mutex_not_held(void)
 766 {
 767 #ifdef CONFIG_PROVE_LOCKING
 768         if (debug_locks)
 769                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 770 #endif
 771 }
 772
 773 static const struct nft_chain_type *
 774 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 775                             u8 family, bool autoload)
 776 {
 777         const struct nft_chain_type *type;
 778
 779         type = __nf_tables_chain_type_lookup(nla, family);
 780         if (type != NULL)
 781                 return type;
 782
 783         lockdep_nfnl_nft_mutex_not_held();
 784 #ifdef CONFIG_MODULES
 785         if (autoload) {
 786                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 787                                        nla_len(nla),
 788                                        (const char *)nla_data(nla)) == -EAGAIN)
 789                         return ERR_PTR(-EAGAIN);
 790         }
 791 #endif
 792         return ERR_PTR(-ENOENT);
 793 }
 794
 795 static __be16 nft_base_seq(const struct net *net)
 796 {
 797         struct nftables_pernet *nft_net = nft_pernet(net);
 798
 799         return htons(nft_net->base_seq & 0xffff);
 800 }
 801
 802 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 803         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 804                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 805         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 806         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 807         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 808                                     .len = NFT_USERDATA_MAXLEN }
 809 };
 810
 811 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 812                                      u32 portid, u32 seq, int event, u32 flags,
 813                                      int family, const struct nft_table *table)
 814 {
 815         struct nlmsghdr *nlh;
 816
 817         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 818         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 819                            NFNETLINK_V0, nft_base_seq(net));
 820         if (!nlh)
 821                 goto nla_put_failure;
 822
 823         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 824             nla_put_be32(skb, NFTA_TABLE_FLAGS,
 825                          htonl(table->flags & NFT_TABLE_F_MASK)) ||
 826             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 827             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 828                          NFTA_TABLE_PAD))
 829                 goto nla_put_failure;
 830         if (nft_table_has_owner(table) &&
 831             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 832                 goto nla_put_failure;
 833
 834         if (table->udata) {
 835                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 836                         goto nla_put_failure;
 837         }
 838
 839         nlmsg_end(skb, nlh);
 840         return 0;
 841
 842 nla_put_failure:
 843         nlmsg_trim(skb, nlh);
 844         return -1;
 845 }
 846
 847 struct nftnl_skb_parms {
 848         bool report;
 849 };
 850 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
 851
 852 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
 853                                struct list_head *notify_list)
 854 {
 855         NFT_CB(skb).report = report;
 856         list_add_tail(&skb->list, notify_list);
 857 }
 858
 859 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
 860 {
 861         struct nftables_pernet *nft_net;
 862         struct sk_buff *skb;
 863         u16 flags = 0;
 864         int err;
 865
 866         if (!ctx->report &&
 867             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 868                 return;
 869
 870         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 871         if (skb == NULL)
 872                 goto err;
 873
 874         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 875                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 876
 877         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
 878                                         event, flags, ctx->family, ctx->table);
 879         if (err < 0) {
 880                 kfree_skb(skb);
 881                 goto err;
 882         }
 883
 884         nft_net = nft_pernet(ctx->net);
 885         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 886         return;
 887 err:
 888         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 889 }
 890
 891 static int nf_tables_dump_tables(struct sk_buff *skb,
 892                                  struct netlink_callback *cb)
 893 {
 894         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 895         struct nftables_pernet *nft_net;
 896         const struct nft_table *table;
 897         unsigned int idx = 0, s_idx = cb->args[0];
 898         struct net *net = sock_net(skb->sk);
 899         int family = nfmsg->nfgen_family;
 900
 901         rcu_read_lock();
 902         nft_net = nft_pernet(net);
 903         cb->seq = READ_ONCE(nft_net->base_seq);
 904
 905         list_for_each_entry_rcu(table, &nft_net->tables, list) {
 906                 if (family != NFPROTO_UNSPEC && family != table->family)
 907                         continue;
 908
 909                 if (idx < s_idx)
 910                         goto cont;
 911                 if (idx > s_idx)
 912                         memset(&cb->args[1], 0,
 913                                sizeof(cb->args) - sizeof(cb->args[0]));
 914                 if (!nft_is_active(net, table))
 915                         continue;
 916                 if (nf_tables_fill_table_info(skb, net,
 917                                               NETLINK_CB(cb->skb).portid,
 918                                               cb->nlh->nlmsg_seq,
 919                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
 920                                               table->family, table) < 0)
 921                         goto done;
 922
 923                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 924 cont:
 925                 idx++;
 926         }
 927 done:
 928         rcu_read_unlock();
 929         cb->args[0] = idx;
 930         return skb->len;
 931 }
 932
 933 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
 934                                       const struct nlmsghdr *nlh,
 935                                       struct netlink_dump_control *c)
 936 {
 937         int err;
 938
 939         if (!try_module_get(THIS_MODULE))
 940                 return -EINVAL;
 941
 942         rcu_read_unlock();
 943         err = netlink_dump_start(nlsk, skb, nlh, c);
 944         rcu_read_lock();
 945         module_put(THIS_MODULE);
 946
 947         return err;
 948 }
 949
 950 /* called with rcu_read_lock held */
 951 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
 952                               const struct nlattr * const nla[])
 953 {
 954         struct netlink_ext_ack *extack = info->extack;
 955         u8 genmask = nft_genmask_cur(info->net);
 956         u8 family = info->nfmsg->nfgen_family;
 957         const struct nft_table *table;
 958         struct net *net = info->net;
 959         struct sk_buff *skb2;
 960         int err;
 961
 962         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 963                 struct netlink_dump_control c = {
 964                         .dump = nf_tables_dump_tables,
 965                         .module = THIS_MODULE,
 966                 };
 967
 968                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 969         }
 970
 971         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
 972         if (IS_ERR(table)) {
 973                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
 974                 return PTR_ERR(table);
 975         }
 976
 977         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 978         if (!skb2)
 979                 return -ENOMEM;
 980
 981         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
 982                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
 983                                         0, family, table);
 984         if (err < 0)
 985                 goto err_fill_table_info;
 986
 987         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
 988
 989 err_fill_table_info:
 990         kfree_skb(skb2);
 991         return err;
 992 }
 993
 994 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
 995 {
 996         struct nft_chain *chain;
 997         u32 i = 0;
 998
 999         list_for_each_entry(chain, &table->chains, list) {
1000                 if (!nft_is_active_next(net, chain))
1001                         continue;
1002                 if (!nft_is_base_chain(chain))
1003                         continue;
1004
1005                 if (cnt && i++ == cnt)
1006                         break;
1007
1008                 nf_tables_unregister_hook(net, table, chain);
1009         }
1010 }
1011
1012 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1013 {
1014         struct nft_chain *chain;
1015         int err, i = 0;
1016
1017         list_for_each_entry(chain, &table->chains, list) {
1018                 if (!nft_is_active_next(net, chain))
1019                         continue;
1020                 if (!nft_is_base_chain(chain))
1021                         continue;
1022
1023                 err = nf_tables_register_hook(net, table, chain);
1024                 if (err < 0)
1025                         goto err_register_hooks;
1026
1027                 i++;
1028         }
1029         return 0;
1030
1031 err_register_hooks:
1032         if (i)
1033                 nft_table_disable(net, table, i);
1034         return err;
1035 }
1036
1037 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1038 {
1039         table->flags &= ~NFT_TABLE_F_DORMANT;
1040         nft_table_disable(net, table, 0);
1041         table->flags |= NFT_TABLE_F_DORMANT;
1042 }
1043
1044 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1045 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1046 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1047 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1048                                          __NFT_TABLE_F_WAS_AWAKEN)
1049
1050 static int nf_tables_updtable(struct nft_ctx *ctx)
1051 {
1052         struct nft_trans *trans;
1053         u32 flags;
1054         int ret;
1055
1056         if (!ctx->nla[NFTA_TABLE_FLAGS])
1057                 return 0;
1058
1059         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1060         if (flags & ~NFT_TABLE_F_MASK)
1061                 return -EOPNOTSUPP;
1062
1063         if (flags == ctx->table->flags)
1064                 return 0;
1065
1066         if ((nft_table_has_owner(ctx->table) &&
1067              !(flags & NFT_TABLE_F_OWNER)) ||
1068             (!nft_table_has_owner(ctx->table) &&
1069              flags & NFT_TABLE_F_OWNER))
1070                 return -EOPNOTSUPP;
1071
1072         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1073                                 sizeof(struct nft_trans_table));
1074         if (trans == NULL)
1075                 return -ENOMEM;
1076
1077         if ((flags & NFT_TABLE_F_DORMANT) &&
1078             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1079                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1080                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1081                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1082         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1083                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1084                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1085                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1086                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1087                         if (ret < 0)
1088                                 goto err_register_hooks;
1089
1090                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1091                 }
1092         }
1093
1094         nft_trans_table_update(trans) = true;
1095         nft_trans_commit_list_add_tail(ctx->net, trans);
1096
1097         return 0;
1098
1099 err_register_hooks:
1100         nft_trans_destroy(trans);
1101         return ret;
1102 }
1103
1104 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1105 {
1106         const char *name = data;
1107
1108         return jhash(name, strlen(name), seed);
1109 }
1110
1111 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1112 {
1113         const struct nft_chain *chain = data;
1114
1115         return nft_chain_hash(chain->name, 0, seed);
1116 }
1117
1118 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1119                               const void *ptr)
1120 {
1121         const struct nft_chain *chain = ptr;
1122         const char *name = arg->key;
1123
1124         return strcmp(chain->name, name);
1125 }
1126
1127 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1128 {
1129         const struct nft_object_hash_key *k = data;
1130
1131         seed ^= hash_ptr(k->table, 32);
1132
1133         return jhash(k->name, strlen(k->name), seed);
1134 }
1135
1136 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1137 {
1138         const struct nft_object *obj = data;
1139
1140         return nft_objname_hash(&obj->key, 0, seed);
1141 }
1142
1143 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1144                                 const void *ptr)
1145 {
1146         const struct nft_object_hash_key *k = arg->key;
1147         const struct nft_object *obj = ptr;
1148
1149         if (obj->key.table != k->table)
1150                 return -1;
1151
1152         return strcmp(obj->key.name, k->name);
1153 }
1154
1155 static bool nft_supported_family(u8 family)
1156 {
1157         return false
1158 #ifdef CONFIG_NF_TABLES_INET
1159                 || family == NFPROTO_INET
1160 #endif
1161 #ifdef CONFIG_NF_TABLES_IPV4
1162                 || family == NFPROTO_IPV4
1163 #endif
1164 #ifdef CONFIG_NF_TABLES_ARP
1165                 || family == NFPROTO_ARP
1166 #endif
1167 #ifdef CONFIG_NF_TABLES_NETDEV
1168                 || family == NFPROTO_NETDEV
1169 #endif
1170 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1171                 || family == NFPROTO_BRIDGE
1172 #endif
1173 #ifdef CONFIG_NF_TABLES_IPV6
1174                 || family == NFPROTO_IPV6
1175 #endif
1176                 ;
1177 }
1178
1179 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1180                               const struct nlattr * const nla[])
1181 {
1182         struct nftables_pernet *nft_net = nft_pernet(info->net);
1183         struct netlink_ext_ack *extack = info->extack;
1184         u8 genmask = nft_genmask_next(info->net);
1185         u8 family = info->nfmsg->nfgen_family;
1186         struct net *net = info->net;
1187         const struct nlattr *attr;
1188         struct nft_table *table;
1189         struct nft_ctx ctx;
1190         u32 flags = 0;
1191         int err;
1192
1193         if (!nft_supported_family(family))
1194                 return -EOPNOTSUPP;
1195
1196         lockdep_assert_held(&nft_net->commit_mutex);
1197         attr = nla[NFTA_TABLE_NAME];
1198         table = nft_table_lookup(net, attr, family, genmask,
1199                                  NETLINK_CB(skb).portid);
1200         if (IS_ERR(table)) {
1201                 if (PTR_ERR(table) != -ENOENT)
1202                         return PTR_ERR(table);
1203         } else {
1204                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1205                         NL_SET_BAD_ATTR(extack, attr);
1206                         return -EEXIST;
1207                 }
1208                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1209                         return -EOPNOTSUPP;
1210
1211                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1212
1213                 return nf_tables_updtable(&ctx);
1214         }
1215
1216         if (nla[NFTA_TABLE_FLAGS]) {
1217                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1218                 if (flags & ~NFT_TABLE_F_MASK)
1219                         return -EOPNOTSUPP;
1220         }
1221
1222         err = -ENOMEM;
1223         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1224         if (table == NULL)
1225                 goto err_kzalloc;
1226
1227         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1228         if (table->name == NULL)
1229                 goto err_strdup;
1230
1231         if (nla[NFTA_TABLE_USERDATA]) {
1232                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1233                 if (table->udata == NULL)
1234                         goto err_table_udata;
1235
1236                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1237         }
1238
1239         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1240         if (err)
1241                 goto err_chain_ht;
1242
1243         INIT_LIST_HEAD(&table->chains);
1244         INIT_LIST_HEAD(&table->sets);
1245         INIT_LIST_HEAD(&table->objects);
1246         INIT_LIST_HEAD(&table->flowtables);
1247         table->family = family;
1248         table->flags = flags;
1249         table->handle = ++nft_net->table_handle;
1250         if (table->flags & NFT_TABLE_F_OWNER)
1251                 table->nlpid = NETLINK_CB(skb).portid;
1252
1253         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1254         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1255         if (err < 0)
1256                 goto err_trans;
1257
1258         list_add_tail_rcu(&table->list, &nft_net->tables);
1259         return 0;
1260 err_trans:
1261         rhltable_destroy(&table->chains_ht);
1262 err_chain_ht:
1263         kfree(table->udata);
1264 err_table_udata:
1265         kfree(table->name);
1266 err_strdup:
1267         kfree(table);
1268 err_kzalloc:
1269         return err;
1270 }
1271
1272 static int nft_flush_table(struct nft_ctx *ctx)
1273 {
1274         struct nft_flowtable *flowtable, *nft;
1275         struct nft_chain *chain, *nc;
1276         struct nft_object *obj, *ne;
1277         struct nft_set *set, *ns;
1278         int err;
1279
1280         list_for_each_entry(chain, &ctx->table->chains, list) {
1281                 if (!nft_is_active_next(ctx->net, chain))
1282                         continue;
1283
1284                 if (nft_chain_is_bound(chain))
1285                         continue;
1286
1287                 ctx->chain = chain;
1288
1289                 err = nft_delrule_by_chain(ctx);
1290                 if (err < 0)
1291                         goto out;
1292         }
1293
1294         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1295                 if (!nft_is_active_next(ctx->net, set))
1296                         continue;
1297
1298                 if (nft_set_is_anonymous(set) &&
1299                     !list_empty(&set->bindings))
1300                         continue;
1301
1302                 err = nft_delset(ctx, set);
1303                 if (err < 0)
1304                         goto out;
1305         }
1306
1307         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1308                 if (!nft_is_active_next(ctx->net, flowtable))
1309                         continue;
1310
1311                 err = nft_delflowtable(ctx, flowtable);
1312                 if (err < 0)
1313                         goto out;
1314         }
1315
1316         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1317                 if (!nft_is_active_next(ctx->net, obj))
1318                         continue;
1319
1320                 err = nft_delobj(ctx, obj);
1321                 if (err < 0)
1322                         goto out;
1323         }
1324
1325         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1326                 if (!nft_is_active_next(ctx->net, chain))
1327                         continue;
1328
1329                 if (nft_chain_is_bound(chain))
1330                         continue;
1331
1332                 ctx->chain = chain;
1333
1334                 err = nft_delchain(ctx);
1335                 if (err < 0)
1336                         goto out;
1337         }
1338
1339         err = nft_deltable(ctx);
1340 out:
1341         return err;
1342 }
1343
1344 static int nft_flush(struct nft_ctx *ctx, int family)
1345 {
1346         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1347         const struct nlattr * const *nla = ctx->nla;
1348         struct nft_table *table, *nt;
1349         int err = 0;
1350
1351         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1352                 if (family != AF_UNSPEC && table->family != family)
1353                         continue;
1354
1355                 ctx->family = table->family;
1356
1357                 if (!nft_is_active_next(ctx->net, table))
1358                         continue;
1359
1360                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1361                         continue;
1362
1363                 if (nla[NFTA_TABLE_NAME] &&
1364                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1365                         continue;
1366
1367                 ctx->table = table;
1368
1369                 err = nft_flush_table(ctx);
1370                 if (err < 0)
1371                         goto out;
1372         }
1373 out:
1374         return err;
1375 }
1376
1377 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1378                               const struct nlattr * const nla[])
1379 {
1380         struct netlink_ext_ack *extack = info->extack;
1381         u8 genmask = nft_genmask_next(info->net);
1382         u8 family = info->nfmsg->nfgen_family;
1383         struct net *net = info->net;
1384         const struct nlattr *attr;
1385         struct nft_table *table;
1386         struct nft_ctx ctx;
1387
1388         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1389         if (family == AF_UNSPEC ||
1390             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1391                 return nft_flush(&ctx, family);
1392
1393         if (nla[NFTA_TABLE_HANDLE]) {
1394                 attr = nla[NFTA_TABLE_HANDLE];
1395                 table = nft_table_lookup_byhandle(net, attr, genmask,
1396                                                   NETLINK_CB(skb).portid);
1397         } else {
1398                 attr = nla[NFTA_TABLE_NAME];
1399                 table = nft_table_lookup(net, attr, family, genmask,
1400                                          NETLINK_CB(skb).portid);
1401         }
1402
1403         if (IS_ERR(table)) {
1404                 NL_SET_BAD_ATTR(extack, attr);
1405                 return PTR_ERR(table);
1406         }
1407
1408         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1409             table->use > 0)
1410                 return -EBUSY;
1411
1412         ctx.family = family;
1413         ctx.table = table;
1414
1415         return nft_flush_table(&ctx);
1416 }
1417
1418 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1419 {
1420         if (WARN_ON(ctx->table->use > 0))
1421                 return;
1422
1423         rhltable_destroy(&ctx->table->chains_ht);
1424         kfree(ctx->table->name);
1425         kfree(ctx->table->udata);
1426         kfree(ctx->table);
1427 }
1428
1429 void nft_register_chain_type(const struct nft_chain_type *ctype)
1430 {
1431         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1432         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1433                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1434                 return;
1435         }
1436         chain_type[ctype->family][ctype->type] = ctype;
1437         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1438 }
1439 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1440
1441 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1442 {
1443         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1444         chain_type[ctype->family][ctype->type] = NULL;
1445         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1446 }
1447 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1448
1449 /*
1450  * Chains
1451  */
1452
1453 static struct nft_chain *
1454 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1455 {
1456         struct nft_chain *chain;
1457
1458         list_for_each_entry(chain, &table->chains, list) {
1459                 if (chain->handle == handle &&
1460                     nft_active_genmask(chain, genmask))
1461                         return chain;
1462         }
1463
1464         return ERR_PTR(-ENOENT);
1465 }
1466
1467 static bool lockdep_commit_lock_is_held(const struct net *net)
1468 {
1469 #ifdef CONFIG_PROVE_LOCKING
1470         struct nftables_pernet *nft_net = nft_pernet(net);
1471
1472         return lockdep_is_held(&nft_net->commit_mutex);
1473 #else
1474         return true;
1475 #endif
1476 }
1477
1478 static struct nft_chain *nft_chain_lookup(struct net *net,
1479                                           struct nft_table *table,
1480                                           const struct nlattr *nla, u8 genmask)
1481 {
1482         char search[NFT_CHAIN_MAXNAMELEN + 1];
1483         struct rhlist_head *tmp, *list;
1484         struct nft_chain *chain;
1485
1486         if (nla == NULL)
1487                 return ERR_PTR(-EINVAL);
1488
1489         nla_strscpy(search, nla, sizeof(search));
1490
1491         WARN_ON(!rcu_read_lock_held() &&
1492                 !lockdep_commit_lock_is_held(net));
1493
1494         chain = ERR_PTR(-ENOENT);
1495         rcu_read_lock();
1496         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1497         if (!list)
1498                 goto out_unlock;
1499
1500         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1501                 if (nft_active_genmask(chain, genmask))
1502                         goto out_unlock;
1503         }
1504         chain = ERR_PTR(-ENOENT);
1505 out_unlock:
1506         rcu_read_unlock();
1507         return chain;
1508 }
1509
1510 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1511         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1512                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1513         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1514         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1515                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1516         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1517         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1518         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1519                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1520         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1521         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1522         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1523         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1524                                     .len = NFT_USERDATA_MAXLEN },
1525 };
1526
1527 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1528         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1529         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1530         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1531                                     .len = IFNAMSIZ - 1 },
1532 };
1533
1534 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1535 {
1536         struct nft_stats *cpu_stats, total;
1537         struct nlattr *nest;
1538         unsigned int seq;
1539         u64 pkts, bytes;
1540         int cpu;
1541
1542         if (!stats)
1543                 return 0;
1544
1545         memset(&total, 0, sizeof(total));
1546         for_each_possible_cpu(cpu) {
1547                 cpu_stats = per_cpu_ptr(stats, cpu);
1548                 do {
1549                         seq = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
1550                         pkts = cpu_stats->pkts;
1551                         bytes = cpu_stats->bytes;
1552                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, seq));
1553                 total.pkts += pkts;
1554                 total.bytes += bytes;
1555         }
1556         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1557         if (nest == NULL)
1558                 goto nla_put_failure;
1559
1560         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1561                          NFTA_COUNTER_PAD) ||
1562             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1563                          NFTA_COUNTER_PAD))
1564                 goto nla_put_failure;
1565
1566         nla_nest_end(skb, nest);
1567         return 0;
1568
1569 nla_put_failure:
1570         return -ENOSPC;
1571 }
1572
1573 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1574                                    const struct nft_base_chain *basechain)
1575 {
1576         const struct nf_hook_ops *ops = &basechain->ops;
1577         struct nft_hook *hook, *first = NULL;
1578         struct nlattr *nest, *nest_devs;
1579         int n = 0;
1580
1581         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1582         if (nest == NULL)
1583                 goto nla_put_failure;
1584         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1585                 goto nla_put_failure;
1586         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1587                 goto nla_put_failure;
1588
1589         if (nft_base_chain_netdev(family, ops->hooknum)) {
1590                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1591                 list_for_each_entry(hook, &basechain->hook_list, list) {
1592                         if (!first)
1593                                 first = hook;
1594
1595                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1596                                            hook->ops.dev->name))
1597                                 goto nla_put_failure;
1598                         n++;
1599                 }
1600                 nla_nest_end(skb, nest_devs);
1601
1602                 if (n == 1 &&
1603                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1604                         goto nla_put_failure;
1605         }
1606         nla_nest_end(skb, nest);
1607
1608         return 0;
1609 nla_put_failure:
1610         return -1;
1611 }
1612
1613 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1614                                      u32 portid, u32 seq, int event, u32 flags,
1615                                      int family, const struct nft_table *table,
1616                                      const struct nft_chain *chain)
1617 {
1618         struct nlmsghdr *nlh;
1619
1620         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1621         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1622                            NFNETLINK_V0, nft_base_seq(net));
1623         if (!nlh)
1624                 goto nla_put_failure;
1625
1626         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name))
1627                 goto nla_put_failure;
1628         if (nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1629                          NFTA_CHAIN_PAD))
1630                 goto nla_put_failure;
1631         if (nla_put_string(skb, NFTA_CHAIN_NAME, chain->name))
1632                 goto nla_put_failure;
1633
1634         if (nft_is_base_chain(chain)) {
1635                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1636                 struct nft_stats __percpu *stats;
1637
1638                 if (nft_dump_basechain_hook(skb, family, basechain))
1639                         goto nla_put_failure;
1640
1641                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1642                                  htonl(basechain->policy)))
1643                         goto nla_put_failure;
1644
1645                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1646                         goto nla_put_failure;
1647
1648                 stats = rcu_dereference_check(basechain->stats,
1649                                               lockdep_commit_lock_is_held(net));
1650                 if (nft_dump_stats(skb, stats))
1651                         goto nla_put_failure;
1652         }
1653
1654         if (chain->flags &&
1655             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1656                 goto nla_put_failure;
1657
1658         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1659                 goto nla_put_failure;
1660
1661         if (chain->udata &&
1662             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1663                 goto nla_put_failure;
1664
1665         nlmsg_end(skb, nlh);
1666         return 0;
1667
1668 nla_put_failure:
1669         nlmsg_trim(skb, nlh);
1670         return -1;
1671 }
1672
1673 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event)
1674 {
1675         struct nftables_pernet *nft_net;
1676         struct sk_buff *skb;
1677         u16 flags = 0;
1678         int err;
1679
1680         if (!ctx->report &&
1681             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1682                 return;
1683
1684         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1685         if (skb == NULL)
1686                 goto err;
1687
1688         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1689                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1690
1691         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1692                                         event, flags, ctx->family, ctx->table,
1693                                         ctx->chain);
1694         if (err < 0) {
1695                 kfree_skb(skb);
1696                 goto err;
1697         }
1698
1699         nft_net = nft_pernet(ctx->net);
1700         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1701         return;
1702 err:
1703         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1704 }
1705
1706 static int nf_tables_dump_chains(struct sk_buff *skb,
1707                                  struct netlink_callback *cb)
1708 {
1709         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1710         unsigned int idx = 0, s_idx = cb->args[0];
1711         struct net *net = sock_net(skb->sk);
1712         int family = nfmsg->nfgen_family;
1713         struct nftables_pernet *nft_net;
1714         const struct nft_table *table;
1715         const struct nft_chain *chain;
1716
1717         rcu_read_lock();
1718         nft_net = nft_pernet(net);
1719         cb->seq = READ_ONCE(nft_net->base_seq);
1720
1721         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1722                 if (family != NFPROTO_UNSPEC && family != table->family)
1723                         continue;
1724
1725                 list_for_each_entry_rcu(chain, &table->chains, list) {
1726                         if (idx < s_idx)
1727                                 goto cont;
1728                         if (idx > s_idx)
1729                                 memset(&cb->args[1], 0,
1730                                        sizeof(cb->args) - sizeof(cb->args[0]));
1731                         if (!nft_is_active(net, chain))
1732                                 continue;
1733                         if (nf_tables_fill_chain_info(skb, net,
1734                                                       NETLINK_CB(cb->skb).portid,
1735                                                       cb->nlh->nlmsg_seq,
1736                                                       NFT_MSG_NEWCHAIN,
1737                                                       NLM_F_MULTI,
1738                                                       table->family, table,
1739                                                       chain) < 0)
1740                                 goto done;
1741
1742                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1743 cont:
1744                         idx++;
1745                 }
1746         }
1747 done:
1748         rcu_read_unlock();
1749         cb->args[0] = idx;
1750         return skb->len;
1751 }
1752
1753 /* called with rcu_read_lock held */
1754 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1755                               const struct nlattr * const nla[])
1756 {
1757         struct netlink_ext_ack *extack = info->extack;
1758         u8 genmask = nft_genmask_cur(info->net);
1759         u8 family = info->nfmsg->nfgen_family;
1760         const struct nft_chain *chain;
1761         struct net *net = info->net;
1762         struct nft_table *table;
1763         struct sk_buff *skb2;
1764         int err;
1765
1766         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1767                 struct netlink_dump_control c = {
1768                         .dump = nf_tables_dump_chains,
1769                         .module = THIS_MODULE,
1770                 };
1771
1772                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1773         }
1774
1775         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1776         if (IS_ERR(table)) {
1777                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1778                 return PTR_ERR(table);
1779         }
1780
1781         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1782         if (IS_ERR(chain)) {
1783                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1784                 return PTR_ERR(chain);
1785         }
1786
1787         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1788         if (!skb2)
1789                 return -ENOMEM;
1790
1791         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1792                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1793                                         0, family, table, chain);
1794         if (err < 0)
1795                 goto err_fill_chain_info;
1796
1797         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1798
1799 err_fill_chain_info:
1800         kfree_skb(skb2);
1801         return err;
1802 }
1803
1804 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1805         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1806         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1807 };
1808
1809 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1810 {
1811         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1812         struct nft_stats __percpu *newstats;
1813         struct nft_stats *stats;
1814         int err;
1815
1816         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1817                                           nft_counter_policy, NULL);
1818         if (err < 0)
1819                 return ERR_PTR(err);
1820
1821         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1822                 return ERR_PTR(-EINVAL);
1823
1824         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1825         if (newstats == NULL)
1826                 return ERR_PTR(-ENOMEM);
1827
1828         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1829          * are not exposed to userspace.
1830          */
1831         preempt_disable();
1832         stats = this_cpu_ptr(newstats);
1833         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
1834         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
1835         preempt_enable();
1836
1837         return newstats;
1838 }
1839
1840 static void nft_chain_stats_replace(struct nft_trans *trans)
1841 {
1842         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
1843
1844         if (!nft_trans_chain_stats(trans))
1845                 return;
1846
1847         nft_trans_chain_stats(trans) =
1848                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
1849                                     lockdep_commit_lock_is_held(trans->ctx.net));
1850
1851         if (!nft_trans_chain_stats(trans))
1852                 static_branch_inc(&nft_counters_enabled);
1853 }
1854
1855 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
1856 {
1857         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
1858         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
1859
1860         if (g0 != g1)
1861                 kvfree(g1);
1862         kvfree(g0);
1863
1864         /* should be NULL either via abort or via successful commit */
1865         WARN_ON_ONCE(chain->blob_next);
1866         kvfree(chain->blob_next);
1867 }
1868
1869 void nf_tables_chain_destroy(struct nft_ctx *ctx)
1870 {
1871         struct nft_chain *chain = ctx->chain;
1872         struct nft_hook *hook, *next;
1873
1874         if (WARN_ON(chain->use > 0))
1875                 return;
1876
1877         /* no concurrent access possible anymore */
1878         nf_tables_chain_free_chain_rules(chain);
1879
1880         if (nft_is_base_chain(chain)) {
1881                 struct nft_base_chain *basechain = nft_base_chain(chain);
1882
1883                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
1884                         list_for_each_entry_safe(hook, next,
1885                                                  &basechain->hook_list, list) {
1886                                 list_del_rcu(&hook->list);
1887                                 kfree_rcu(hook, rcu);
1888                         }
1889                 }
1890                 module_put(basechain->type->owner);
1891                 if (rcu_access_pointer(basechain->stats)) {
1892                         static_branch_dec(&nft_counters_enabled);
1893                         free_percpu(rcu_dereference_raw(basechain->stats));
1894                 }
1895                 kfree(chain->name);
1896                 kfree(chain->udata);
1897                 kfree(basechain);
1898         } else {
1899                 kfree(chain->name);
1900                 kfree(chain->udata);
1901                 kfree(chain);
1902         }
1903 }
1904
1905 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
1906                                               const struct nlattr *attr)
1907 {
1908         struct net_device *dev;
1909         char ifname[IFNAMSIZ];
1910         struct nft_hook *hook;
1911         int err;
1912
1913         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
1914         if (!hook) {
1915                 err = -ENOMEM;
1916                 goto err_hook_alloc;
1917         }
1918
1919         nla_strscpy(ifname, attr, IFNAMSIZ);
1920         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
1921          * indirectly serializing all the other holders of the commit_mutex with
1922          * the rtnl_mutex.
1923          */
1924         dev = __dev_get_by_name(net, ifname);
1925         if (!dev) {
1926                 err = -ENOENT;
1927                 goto err_hook_dev;
1928         }
1929         hook->ops.dev = dev;
1930
1931         return hook;
1932
1933 err_hook_dev:
1934         kfree(hook);
1935 err_hook_alloc:
1936         return ERR_PTR(err);
1937 }
1938
1939 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
1940                                            const struct nft_hook *this)
1941 {
1942         struct nft_hook *hook;
1943
1944         list_for_each_entry(hook, hook_list, list) {
1945                 if (this->ops.dev == hook->ops.dev)
1946                         return hook;
1947         }
1948
1949         return NULL;
1950 }
1951
1952 static int nf_tables_parse_netdev_hooks(struct net *net,
1953                                         const struct nlattr *attr,
1954                                         struct list_head *hook_list)
1955 {
1956         struct nft_hook *hook, *next;
1957         const struct nlattr *tmp;
1958         int rem, n = 0, err;
1959
1960         nla_for_each_nested(tmp, attr, rem) {
1961                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
1962                         err = -EINVAL;
1963                         goto err_hook;
1964                 }
1965
1966                 hook = nft_netdev_hook_alloc(net, tmp);
1967                 if (IS_ERR(hook)) {
1968                         err = PTR_ERR(hook);
1969                         goto err_hook;
1970                 }
1971                 if (nft_hook_list_find(hook_list, hook)) {
1972                         kfree(hook);
1973                         err = -EEXIST;
1974                         goto err_hook;
1975                 }
1976                 list_add_tail(&hook->list, hook_list);
1977                 n++;
1978
1979                 if (n == NFT_NETDEVICE_MAX) {
1980                         err = -EFBIG;
1981                         goto err_hook;
1982                 }
1983         }
1984
1985         return 0;
1986
1987 err_hook:
1988         list_for_each_entry_safe(hook, next, hook_list, list) {
1989                 list_del(&hook->list);
1990                 kfree(hook);
1991         }
1992         return err;
1993 }
1994
1995 struct nft_chain_hook {
1996         u32                             num;
1997         s32                             priority;
1998         const struct nft_chain_type     *type;
1999         struct list_head                list;
2000 };
2001
2002 static int nft_chain_parse_netdev(struct net *net,
2003                                   struct nlattr *tb[],
2004                                   struct list_head *hook_list)
2005 {
2006         struct nft_hook *hook;
2007         int err;
2008
2009         if (tb[NFTA_HOOK_DEV]) {
2010                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2011                 if (IS_ERR(hook))
2012                         return PTR_ERR(hook);
2013
2014                 list_add_tail(&hook->list, hook_list);
2015         } else if (tb[NFTA_HOOK_DEVS]) {
2016                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2017                                                    hook_list);
2018                 if (err < 0)
2019                         return err;
2020
2021                 if (list_empty(hook_list))
2022                         return -EINVAL;
2023         } else {
2024                 return -EINVAL;
2025         }
2026
2027         return 0;
2028 }
2029
2030 static int nft_chain_parse_hook(struct net *net,
2031                                 const struct nlattr * const nla[],
2032                                 struct nft_chain_hook *hook, u8 family,
2033                                 struct netlink_ext_ack *extack, bool autoload)
2034 {
2035         struct nftables_pernet *nft_net = nft_pernet(net);
2036         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2037         const struct nft_chain_type *type;
2038         int err;
2039
2040         lockdep_assert_held(&nft_net->commit_mutex);
2041         lockdep_nfnl_nft_mutex_not_held();
2042
2043         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2044                                           nla[NFTA_CHAIN_HOOK],
2045                                           nft_hook_policy, NULL);
2046         if (err < 0)
2047                 return err;
2048
2049         if (ha[NFTA_HOOK_HOOKNUM] == NULL ||
2050             ha[NFTA_HOOK_PRIORITY] == NULL)
2051                 return -EINVAL;
2052
2053         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2054         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2055
2056         type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2057         if (!type)
2058                 return -EOPNOTSUPP;
2059
2060         if (nla[NFTA_CHAIN_TYPE]) {
2061                 type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2062                                                    family, autoload);
2063                 if (IS_ERR(type)) {
2064                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2065                         return PTR_ERR(type);
2066                 }
2067         }
2068         if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2069                 return -EOPNOTSUPP;
2070
2071         if (type->type == NFT_CHAIN_T_NAT &&
2072             hook->priority <= NF_IP_PRI_CONNTRACK)
2073                 return -EOPNOTSUPP;
2074
2075         if (!try_module_get(type->owner)) {
2076                 if (nla[NFTA_CHAIN_TYPE])
2077                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2078                 return -ENOENT;
2079         }
2080
2081         hook->type = type;
2082
2083         INIT_LIST_HEAD(&hook->list);
2084         if (nft_base_chain_netdev(family, hook->num)) {
2085                 err = nft_chain_parse_netdev(net, ha, &hook->list);
2086                 if (err < 0) {
2087                         module_put(type->owner);
2088                         return err;
2089                 }
2090         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2091                 module_put(type->owner);
2092                 return -EOPNOTSUPP;
2093         }
2094
2095         return 0;
2096 }
2097
2098 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2099 {
2100         struct nft_hook *h, *next;
2101
2102         list_for_each_entry_safe(h, next, &hook->list, list) {
2103                 list_del(&h->list);
2104                 kfree(h);
2105         }
2106         module_put(hook->type->owner);
2107 }
2108
2109 struct nft_rules_old {
2110         struct rcu_head h;
2111         struct nft_rule_blob *blob;
2112 };
2113
2114 static void nft_last_rule(struct nft_rule_blob *blob, const void *ptr)
2115 {
2116         struct nft_rule_dp *prule;
2117
2118         prule = (struct nft_rule_dp *)ptr;
2119         prule->is_last = 1;
2120         /* blob size does not include the trailer rule */
2121 }
2122
2123 static struct nft_rule_blob *nf_tables_chain_alloc_rules(unsigned int size)
2124 {
2125         struct nft_rule_blob *blob;
2126
2127         /* size must include room for the last rule */
2128         if (size < offsetof(struct nft_rule_dp, data))
2129                 return NULL;
2130
2131         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rules_old);
2132         if (size > INT_MAX)
2133                 return NULL;
2134
2135         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2136         if (!blob)
2137                 return NULL;
2138
2139         blob->size = 0;
2140         nft_last_rule(blob, blob->data);
2141
2142         return blob;
2143 }
2144
2145 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2146                                     const struct nft_chain_hook *hook,
2147                                     struct nft_chain *chain)
2148 {
2149         ops->pf                 = family;
2150         ops->hooknum            = hook->num;
2151         ops->priority           = hook->priority;
2152         ops->priv               = chain;
2153         ops->hook               = hook->type->hooks[ops->hooknum];
2154         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2155 }
2156
2157 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2158                               struct nft_chain_hook *hook, u32 flags)
2159 {
2160         struct nft_chain *chain;
2161         struct nft_hook *h;
2162
2163         basechain->type = hook->type;
2164         INIT_LIST_HEAD(&basechain->hook_list);
2165         chain = &basechain->chain;
2166
2167         if (nft_base_chain_netdev(family, hook->num)) {
2168                 list_splice_init(&hook->list, &basechain->hook_list);
2169                 list_for_each_entry(h, &basechain->hook_list, list)
2170                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2171
2172                 basechain->ops.hooknum  = hook->num;
2173                 basechain->ops.priority = hook->priority;
2174         } else {
2175                 nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2176         }
2177
2178         chain->flags |= NFT_CHAIN_BASE | flags;
2179         basechain->policy = NF_ACCEPT;
2180         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2181             !nft_chain_offload_support(basechain)) {
2182                 list_splice_init(&basechain->hook_list, &hook->list);
2183                 return -EOPNOTSUPP;
2184         }
2185
2186         flow_block_init(&basechain->flow_block);
2187
2188         return 0;
2189 }
2190
2191 static int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2192 {
2193         int err;
2194
2195         err = rhltable_insert_key(&table->chains_ht, chain->name,
2196                                   &chain->rhlhead, nft_chain_ht_params);
2197         if (err)
2198                 return err;
2199
2200         list_add_tail_rcu(&chain->list, &table->chains);
2201
2202         return 0;
2203 }
2204
2205 static u64 chain_id;
2206
2207 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2208                               u8 policy, u32 flags,
2209                               struct netlink_ext_ack *extack)
2210 {
2211         const struct nlattr * const *nla = ctx->nla;
2212         struct nft_table *table = ctx->table;
2213         struct nft_base_chain *basechain;
2214         struct net *net = ctx->net;
2215         char name[NFT_NAME_MAXLEN];
2216         struct nft_rule_blob *blob;
2217         struct nft_trans *trans;
2218         struct nft_chain *chain;
2219         unsigned int data_size;
2220         int err;
2221
2222         if (table->use == UINT_MAX)
2223                 return -EOVERFLOW;
2224
2225         if (nla[NFTA_CHAIN_HOOK]) {
2226                 struct nft_stats __percpu *stats = NULL;
2227                 struct nft_chain_hook hook;
2228
2229                 if (flags & NFT_CHAIN_BINDING)
2230                         return -EOPNOTSUPP;
2231
2232                 err = nft_chain_parse_hook(net, nla, &hook, family, extack,
2233                                            true);
2234                 if (err < 0)
2235                         return err;
2236
2237                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2238                 if (basechain == NULL) {
2239                         nft_chain_release_hook(&hook);
2240                         return -ENOMEM;
2241                 }
2242                 chain = &basechain->chain;
2243
2244                 if (nla[NFTA_CHAIN_COUNTERS]) {
2245                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2246                         if (IS_ERR(stats)) {
2247                                 nft_chain_release_hook(&hook);
2248                                 kfree(basechain);
2249                                 return PTR_ERR(stats);
2250                         }
2251                         rcu_assign_pointer(basechain->stats, stats);
2252                 }
2253
2254                 err = nft_basechain_init(basechain, family, &hook, flags);
2255                 if (err < 0) {
2256                         nft_chain_release_hook(&hook);
2257                         kfree(basechain);
2258                         free_percpu(stats);
2259                         return err;
2260                 }
2261                 if (stats)
2262                         static_branch_inc(&nft_counters_enabled);
2263         } else {
2264                 if (flags & NFT_CHAIN_BASE)
2265                         return -EINVAL;
2266                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2267                         return -EOPNOTSUPP;
2268
2269                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2270                 if (chain == NULL)
2271                         return -ENOMEM;
2272
2273                 chain->flags = flags;
2274         }
2275         ctx->chain = chain;
2276
2277         INIT_LIST_HEAD(&chain->rules);
2278         chain->handle = nf_tables_alloc_handle(table);
2279         chain->table = table;
2280
2281         if (nla[NFTA_CHAIN_NAME]) {
2282                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2283         } else {
2284                 if (!(flags & NFT_CHAIN_BINDING)) {
2285                         err = -EINVAL;
2286                         goto err_destroy_chain;
2287                 }
2288
2289                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2290                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2291         }
2292
2293         if (!chain->name) {
2294                 err = -ENOMEM;
2295                 goto err_destroy_chain;
2296         }
2297
2298         if (nla[NFTA_CHAIN_USERDATA]) {
2299                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2300                 if (chain->udata == NULL) {
2301                         err = -ENOMEM;
2302                         goto err_destroy_chain;
2303                 }
2304                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2305         }
2306
2307         data_size = offsetof(struct nft_rule_dp, data); /* last rule */
2308         blob = nf_tables_chain_alloc_rules(data_size);
2309         if (!blob) {
2310                 err = -ENOMEM;
2311                 goto err_destroy_chain;
2312         }
2313
2314         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2315         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2316
2317         err = nf_tables_register_hook(net, table, chain);
2318         if (err < 0)
2319                 goto err_destroy_chain;
2320
2321         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2322         if (IS_ERR(trans)) {
2323                 err = PTR_ERR(trans);
2324                 goto err_unregister_hook;
2325         }
2326
2327         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2328         if (nft_is_base_chain(chain))
2329                 nft_trans_chain_policy(trans) = policy;
2330
2331         err = nft_chain_add(table, chain);
2332         if (err < 0) {
2333                 nft_trans_destroy(trans);
2334                 goto err_unregister_hook;
2335         }
2336
2337         table->use++;
2338
2339         return 0;
2340 err_unregister_hook:
2341         nf_tables_unregister_hook(net, table, chain);
2342 err_destroy_chain:
2343         nf_tables_chain_destroy(ctx);
2344
2345         return err;
2346 }
2347
2348 static bool nft_hook_list_equal(struct list_head *hook_list1,
2349                                 struct list_head *hook_list2)
2350 {
2351         struct nft_hook *hook;
2352         int n = 0, m = 0;
2353
2354         n = 0;
2355         list_for_each_entry(hook, hook_list2, list) {
2356                 if (!nft_hook_list_find(hook_list1, hook))
2357                         return false;
2358
2359                 n++;
2360         }
2361         list_for_each_entry(hook, hook_list1, list)
2362                 m++;
2363
2364         return n == m;
2365 }
2366
2367 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2368                               u32 flags, const struct nlattr *attr,
2369                               struct netlink_ext_ack *extack)
2370 {
2371         const struct nlattr * const *nla = ctx->nla;
2372         struct nft_table *table = ctx->table;
2373         struct nft_chain *chain = ctx->chain;
2374         struct nft_base_chain *basechain;
2375         struct nft_stats *stats = NULL;
2376         struct nft_chain_hook hook;
2377         struct nf_hook_ops *ops;
2378         struct nft_trans *trans;
2379         int err;
2380
2381         if (chain->flags ^ flags)
2382                 return -EOPNOTSUPP;
2383
2384         if (nla[NFTA_CHAIN_HOOK]) {
2385                 if (!nft_is_base_chain(chain)) {
2386                         NL_SET_BAD_ATTR(extack, attr);
2387                         return -EEXIST;
2388                 }
2389                 err = nft_chain_parse_hook(ctx->net, nla, &hook, ctx->family,
2390                                            extack, false);
2391                 if (err < 0)
2392                         return err;
2393
2394                 basechain = nft_base_chain(chain);
2395                 if (basechain->type != hook.type) {
2396                         nft_chain_release_hook(&hook);
2397                         NL_SET_BAD_ATTR(extack, attr);
2398                         return -EEXIST;
2399                 }
2400
2401                 if (nft_base_chain_netdev(ctx->family, hook.num)) {
2402                         if (!nft_hook_list_equal(&basechain->hook_list,
2403                                                  &hook.list)) {
2404                                 nft_chain_release_hook(&hook);
2405                                 NL_SET_BAD_ATTR(extack, attr);
2406                                 return -EEXIST;
2407                         }
2408                 } else {
2409                         ops = &basechain->ops;
2410                         if (ops->hooknum != hook.num ||
2411                             ops->priority != hook.priority) {
2412                                 nft_chain_release_hook(&hook);
2413                                 NL_SET_BAD_ATTR(extack, attr);
2414                                 return -EEXIST;
2415                         }
2416                 }
2417                 nft_chain_release_hook(&hook);
2418         }
2419
2420         if (nla[NFTA_CHAIN_HANDLE] &&
2421             nla[NFTA_CHAIN_NAME]) {
2422                 struct nft_chain *chain2;
2423
2424                 chain2 = nft_chain_lookup(ctx->net, table,
2425                                           nla[NFTA_CHAIN_NAME], genmask);
2426                 if (!IS_ERR(chain2)) {
2427                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2428                         return -EEXIST;
2429                 }
2430         }
2431
2432         if (nla[NFTA_CHAIN_COUNTERS]) {
2433                 if (!nft_is_base_chain(chain))
2434                         return -EOPNOTSUPP;
2435
2436                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2437                 if (IS_ERR(stats))
2438                         return PTR_ERR(stats);
2439         }
2440
2441         err = -ENOMEM;
2442         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2443                                 sizeof(struct nft_trans_chain));
2444         if (trans == NULL)
2445                 goto err;
2446
2447         nft_trans_chain_stats(trans) = stats;
2448         nft_trans_chain_update(trans) = true;
2449
2450         if (nla[NFTA_CHAIN_POLICY])
2451                 nft_trans_chain_policy(trans) = policy;
2452         else
2453                 nft_trans_chain_policy(trans) = -1;
2454
2455         if (nla[NFTA_CHAIN_HANDLE] &&
2456             nla[NFTA_CHAIN_NAME]) {
2457                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2458                 struct nft_trans *tmp;
2459                 char *name;
2460
2461                 err = -ENOMEM;
2462                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2463                 if (!name)
2464                         goto err;
2465
2466                 err = -EEXIST;
2467                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2468                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2469                             tmp->ctx.table == table &&
2470                             nft_trans_chain_update(tmp) &&
2471                             nft_trans_chain_name(tmp) &&
2472                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2473                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2474                                 kfree(name);
2475                                 goto err;
2476                         }
2477                 }
2478
2479                 nft_trans_chain_name(trans) = name;
2480         }
2481         nft_trans_commit_list_add_tail(ctx->net, trans);
2482
2483         return 0;
2484 err:
2485         free_percpu(stats);
2486         kfree(trans);
2487         return err;
2488 }
2489
2490 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2491                                                const struct nft_table *table,
2492                                                const struct nlattr *nla)
2493 {
2494         struct nftables_pernet *nft_net = nft_pernet(net);
2495         u32 id = ntohl(nla_get_be32(nla));
2496         struct nft_trans *trans;
2497
2498         list_for_each_entry(trans, &nft_net->commit_list, list) {
2499                 struct nft_chain *chain = trans->ctx.chain;
2500
2501                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2502                     chain->table == table &&
2503                     id == nft_trans_chain_id(trans))
2504                         return chain;
2505         }
2506         return ERR_PTR(-ENOENT);
2507 }
2508
2509 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2510                               const struct nlattr * const nla[])
2511 {
2512         struct nftables_pernet *nft_net = nft_pernet(info->net);
2513         struct netlink_ext_ack *extack = info->extack;
2514         u8 genmask = nft_genmask_next(info->net);
2515         u8 family = info->nfmsg->nfgen_family;
2516         struct nft_chain *chain = NULL;
2517         struct net *net = info->net;
2518         const struct nlattr *attr;
2519         struct nft_table *table;
2520         u8 policy = NF_ACCEPT;
2521         struct nft_ctx ctx;
2522         u64 handle = 0;
2523         u32 flags = 0;
2524
2525         lockdep_assert_held(&nft_net->commit_mutex);
2526
2527         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2528                                  NETLINK_CB(skb).portid);
2529         if (IS_ERR(table)) {
2530                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2531                 return PTR_ERR(table);
2532         }
2533
2534         chain = NULL;
2535         attr = nla[NFTA_CHAIN_NAME];
2536
2537         if (nla[NFTA_CHAIN_HANDLE]) {
2538                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2539                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2540                 if (IS_ERR(chain)) {
2541                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2542                         return PTR_ERR(chain);
2543                 }
2544                 attr = nla[NFTA_CHAIN_HANDLE];
2545         } else if (nla[NFTA_CHAIN_NAME]) {
2546                 chain = nft_chain_lookup(net, table, attr, genmask);
2547                 if (IS_ERR(chain)) {
2548                         if (PTR_ERR(chain) != -ENOENT) {
2549                                 NL_SET_BAD_ATTR(extack, attr);
2550                                 return PTR_ERR(chain);
2551                         }
2552                         chain = NULL;
2553                 }
2554         } else if (!nla[NFTA_CHAIN_ID]) {
2555                 return -EINVAL;
2556         }
2557
2558         if (nla[NFTA_CHAIN_POLICY]) {
2559                 if (chain != NULL &&
2560                     !nft_is_base_chain(chain)) {
2561                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2562                         return -EOPNOTSUPP;
2563                 }
2564
2565                 if (chain == NULL &&
2566                     nla[NFTA_CHAIN_HOOK] == NULL) {
2567                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2568                         return -EOPNOTSUPP;
2569                 }
2570
2571                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2572                 switch (policy) {
2573                 case NF_DROP:
2574                 case NF_ACCEPT:
2575                         break;
2576                 default:
2577                         return -EINVAL;
2578                 }
2579         }
2580
2581         if (nla[NFTA_CHAIN_FLAGS])
2582                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2583         else if (chain)
2584                 flags = chain->flags;
2585
2586         if (flags & ~NFT_CHAIN_FLAGS)
2587                 return -EOPNOTSUPP;
2588
2589         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2590
2591         if (chain != NULL) {
2592                 if (chain->flags & NFT_CHAIN_BINDING)
2593                         return -EINVAL;
2594
2595                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2596                         NL_SET_BAD_ATTR(extack, attr);
2597                         return -EEXIST;
2598                 }
2599                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2600                         return -EOPNOTSUPP;
2601
2602                 flags |= chain->flags & NFT_CHAIN_BASE;
2603                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2604                                           extack);
2605         }
2606
2607         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2608 }
2609
2610 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2611                               const struct nlattr * const nla[])
2612 {
2613         struct netlink_ext_ack *extack = info->extack;
2614         u8 genmask = nft_genmask_next(info->net);
2615         u8 family = info->nfmsg->nfgen_family;
2616         struct net *net = info->net;
2617         const struct nlattr *attr;
2618         struct nft_table *table;
2619         struct nft_chain *chain;
2620         struct nft_rule *rule;
2621         struct nft_ctx ctx;
2622         u64 handle;
2623         u32 use;
2624         int err;
2625
2626         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2627                                  NETLINK_CB(skb).portid);
2628         if (IS_ERR(table)) {
2629                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2630                 return PTR_ERR(table);
2631         }
2632
2633         if (nla[NFTA_CHAIN_HANDLE]) {
2634                 attr = nla[NFTA_CHAIN_HANDLE];
2635                 handle = be64_to_cpu(nla_get_be64(attr));
2636                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2637         } else {
2638                 attr = nla[NFTA_CHAIN_NAME];
2639                 chain = nft_chain_lookup(net, table, attr, genmask);
2640         }
2641         if (IS_ERR(chain)) {
2642                 NL_SET_BAD_ATTR(extack, attr);
2643                 return PTR_ERR(chain);
2644         }
2645
2646         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2647             chain->use > 0)
2648                 return -EBUSY;
2649
2650         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2651
2652         use = chain->use;
2653         list_for_each_entry(rule, &chain->rules, list) {
2654                 if (!nft_is_active_next(net, rule))
2655                         continue;
2656                 use--;
2657
2658                 err = nft_delrule(&ctx, rule);
2659                 if (err < 0)
2660                         return err;
2661         }
2662
2663         /* There are rules and elements that are still holding references to us,
2664          * we cannot do a recursive removal in this case.
2665          */
2666         if (use > 0) {
2667                 NL_SET_BAD_ATTR(extack, attr);
2668                 return -EBUSY;
2669         }
2670
2671         return nft_delchain(&ctx);
2672 }
2673
2674 /*
2675  * Expressions
2676  */
2677
2678 /**
2679  *      nft_register_expr - register nf_tables expr type
2680  *      @type: expr type
2681  *
2682  *      Registers the expr type for use with nf_tables. Returns zero on
2683  *      success or a negative errno code otherwise.
2684  */
2685 int nft_register_expr(struct nft_expr_type *type)
2686 {
2687         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2688         if (type->family == NFPROTO_UNSPEC)
2689                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2690         else
2691                 list_add_rcu(&type->list, &nf_tables_expressions);
2692         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2693         return 0;
2694 }
2695 EXPORT_SYMBOL_GPL(nft_register_expr);
2696
2697 /**
2698  *      nft_unregister_expr - unregister nf_tables expr type
2699  *      @type: expr type
2700  *
2701  *      Unregisters the expr typefor use with nf_tables.
2702  */
2703 void nft_unregister_expr(struct nft_expr_type *type)
2704 {
2705         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2706         list_del_rcu(&type->list);
2707         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2708 }
2709 EXPORT_SYMBOL_GPL(nft_unregister_expr);
2710
2711 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
2712                                                        struct nlattr *nla)
2713 {
2714         const struct nft_expr_type *type, *candidate = NULL;
2715
2716         list_for_each_entry(type, &nf_tables_expressions, list) {
2717                 if (!nla_strcmp(nla, type->name)) {
2718                         if (!type->family && !candidate)
2719                                 candidate = type;
2720                         else if (type->family == family)
2721                                 candidate = type;
2722                 }
2723         }
2724         return candidate;
2725 }
2726
2727 #ifdef CONFIG_MODULES
2728 static int nft_expr_type_request_module(struct net *net, u8 family,
2729                                         struct nlattr *nla)
2730 {
2731         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
2732                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
2733                 return -EAGAIN;
2734
2735         return 0;
2736 }
2737 #endif
2738
2739 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
2740                                                      u8 family,
2741                                                      struct nlattr *nla)
2742 {
2743         const struct nft_expr_type *type;
2744
2745         if (nla == NULL)
2746                 return ERR_PTR(-EINVAL);
2747
2748         type = __nft_expr_type_get(family, nla);
2749         if (type != NULL && try_module_get(type->owner))
2750                 return type;
2751
2752         lockdep_nfnl_nft_mutex_not_held();
2753 #ifdef CONFIG_MODULES
2754         if (type == NULL) {
2755                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
2756                         return ERR_PTR(-EAGAIN);
2757
2758                 if (nft_request_module(net, "nft-expr-%.*s",
2759                                        nla_len(nla),
2760                                        (char *)nla_data(nla)) == -EAGAIN)
2761                         return ERR_PTR(-EAGAIN);
2762         }
2763 #endif
2764         return ERR_PTR(-ENOENT);
2765 }
2766
2767 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
2768         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
2769                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
2770         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
2771 };
2772
2773 static int nf_tables_fill_expr_info(struct sk_buff *skb,
2774                                     const struct nft_expr *expr)
2775 {
2776         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
2777                 goto nla_put_failure;
2778
2779         if (expr->ops->dump) {
2780                 struct nlattr *data = nla_nest_start_noflag(skb,
2781                                                             NFTA_EXPR_DATA);
2782                 if (data == NULL)
2783                         goto nla_put_failure;
2784                 if (expr->ops->dump(skb, expr) < 0)
2785                         goto nla_put_failure;
2786                 nla_nest_end(skb, data);
2787         }
2788
2789         return skb->len;
2790
2791 nla_put_failure:
2792         return -1;
2793 };
2794
2795 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
2796                   const struct nft_expr *expr)
2797 {
2798         struct nlattr *nest;
2799
2800         nest = nla_nest_start_noflag(skb, attr);
2801         if (!nest)
2802                 goto nla_put_failure;
2803         if (nf_tables_fill_expr_info(skb, expr) < 0)
2804                 goto nla_put_failure;
2805         nla_nest_end(skb, nest);
2806         return 0;
2807
2808 nla_put_failure:
2809         return -1;
2810 }
2811
2812 struct nft_expr_info {
2813         const struct nft_expr_ops       *ops;
2814         const struct nlattr             *attr;
2815         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
2816 };
2817
2818 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
2819                                 const struct nlattr *nla,
2820                                 struct nft_expr_info *info)
2821 {
2822         const struct nft_expr_type *type;
2823         const struct nft_expr_ops *ops;
2824         struct nlattr *tb[NFTA_EXPR_MAX + 1];
2825         int err;
2826
2827         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
2828                                           nft_expr_policy, NULL);
2829         if (err < 0)
2830                 return err;
2831
2832         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
2833         if (IS_ERR(type))
2834                 return PTR_ERR(type);
2835
2836         if (tb[NFTA_EXPR_DATA]) {
2837                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
2838                                                   tb[NFTA_EXPR_DATA],
2839                                                   type->policy, NULL);
2840                 if (err < 0)
2841                         goto err1;
2842         } else
2843                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
2844
2845         if (type->select_ops != NULL) {
2846                 ops = type->select_ops(ctx,
2847                                        (const struct nlattr * const *)info->tb);
2848                 if (IS_ERR(ops)) {
2849                         err = PTR_ERR(ops);
2850 #ifdef CONFIG_MODULES
2851                         if (err == -EAGAIN)
2852                                 if (nft_expr_type_request_module(ctx->net,
2853                                                                  ctx->family,
2854                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
2855                                         err = -ENOENT;
2856 #endif
2857                         goto err1;
2858                 }
2859         } else
2860                 ops = type->ops;
2861
2862         info->attr = nla;
2863         info->ops = ops;
2864
2865         return 0;
2866
2867 err1:
2868         module_put(type->owner);
2869         return err;
2870 }
2871
2872 static int nf_tables_newexpr(const struct nft_ctx *ctx,
2873                              const struct nft_expr_info *expr_info,
2874                              struct nft_expr *expr)
2875 {
2876         const struct nft_expr_ops *ops = expr_info->ops;
2877         int err;
2878
2879         expr->ops = ops;
2880         if (ops->init) {
2881                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
2882                 if (err < 0)
2883                         goto err1;
2884         }
2885
2886         return 0;
2887 err1:
2888         expr->ops = NULL;
2889         return err;
2890 }
2891
2892 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
2893                                    struct nft_expr *expr)
2894 {
2895         const struct nft_expr_type *type = expr->ops->type;
2896
2897         if (expr->ops->destroy)
2898                 expr->ops->destroy(ctx, expr);
2899         module_put(type->owner);
2900 }
2901
2902 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
2903                                       const struct nlattr *nla)
2904 {
2905         struct nft_expr_info expr_info;
2906         struct nft_expr *expr;
2907         struct module *owner;
2908         int err;
2909
2910         err = nf_tables_expr_parse(ctx, nla, &expr_info);
2911         if (err < 0)
2912                 goto err_expr_parse;
2913
2914         err = -EOPNOTSUPP;
2915         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
2916                 goto err_expr_stateful;
2917
2918         err = -ENOMEM;
2919         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
2920         if (expr == NULL)
2921                 goto err_expr_stateful;
2922
2923         err = nf_tables_newexpr(ctx, &expr_info, expr);
2924         if (err < 0)
2925                 goto err_expr_new;
2926
2927         return expr;
2928 err_expr_new:
2929         kfree(expr);
2930 err_expr_stateful:
2931         owner = expr_info.ops->type->owner;
2932         if (expr_info.ops->type->release_ops)
2933                 expr_info.ops->type->release_ops(expr_info.ops);
2934
2935         module_put(owner);
2936 err_expr_parse:
2937         return ERR_PTR(err);
2938 }
2939
2940 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
2941 {
2942         int err;
2943
2944         if (src->ops->clone) {
2945                 dst->ops = src->ops;
2946                 err = src->ops->clone(dst, src);
2947                 if (err < 0)
2948                         return err;
2949         } else {
2950                 memcpy(dst, src, src->ops->size);
2951         }
2952
2953         __module_get(src->ops->type->owner);
2954
2955         return 0;
2956 }
2957
2958 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
2959 {
2960         nf_tables_expr_destroy(ctx, expr);
2961         kfree(expr);
2962 }
2963
2964 /*
2965  * Rules
2966  */
2967
2968 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
2969                                           u64 handle)
2970 {
2971         struct nft_rule *rule;
2972
2973         // FIXME: this sucks
2974         list_for_each_entry_rcu(rule, &chain->rules, list) {
2975                 if (handle == rule->handle)
2976                         return rule;
2977         }
2978
2979         return ERR_PTR(-ENOENT);
2980 }
2981
2982 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
2983                                         const struct nlattr *nla)
2984 {
2985         if (nla == NULL)
2986                 return ERR_PTR(-EINVAL);
2987
2988         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
2989 }
2990
2991 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
2992         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
2993                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
2994         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
2995                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
2996         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
2997         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
2998         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
2999         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3000         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3001                                     .len = NFT_USERDATA_MAXLEN },
3002         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3003         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3004         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3005 };
3006
3007 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3008                                     u32 portid, u32 seq, int event,
3009                                     u32 flags, int family,
3010                                     const struct nft_table *table,
3011                                     const struct nft_chain *chain,
3012                                     const struct nft_rule *rule, u64 handle)
3013 {
3014         struct nlmsghdr *nlh;
3015         const struct nft_expr *expr, *next;
3016         struct nlattr *list;
3017         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3018
3019         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3020                            nft_base_seq(net));
3021         if (!nlh)
3022                 goto nla_put_failure;
3023
3024         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3025                 goto nla_put_failure;
3026         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3027                 goto nla_put_failure;
3028         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3029                          NFTA_RULE_PAD))
3030                 goto nla_put_failure;
3031
3032         if (event != NFT_MSG_DELRULE && handle) {
3033                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3034                                  NFTA_RULE_PAD))
3035                         goto nla_put_failure;
3036         }
3037
3038         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3039                 nft_flow_rule_stats(chain, rule);
3040
3041         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3042         if (list == NULL)
3043                 goto nla_put_failure;
3044         nft_rule_for_each_expr(expr, next, rule) {
3045                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr) < 0)
3046                         goto nla_put_failure;
3047         }
3048         nla_nest_end(skb, list);
3049
3050         if (rule->udata) {
3051                 struct nft_userdata *udata = nft_userdata(rule);
3052                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3053                             udata->data) < 0)
3054                         goto nla_put_failure;
3055         }
3056
3057         nlmsg_end(skb, nlh);
3058         return 0;
3059
3060 nla_put_failure:
3061         nlmsg_trim(skb, nlh);
3062         return -1;
3063 }
3064
3065 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3066                                   const struct nft_rule *rule, int event)
3067 {
3068         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3069         const struct nft_rule *prule;
3070         struct sk_buff *skb;
3071         u64 handle = 0;
3072         u16 flags = 0;
3073         int err;
3074
3075         if (!ctx->report &&
3076             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3077                 return;
3078
3079         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3080         if (skb == NULL)
3081                 goto err;
3082
3083         if (event == NFT_MSG_NEWRULE &&
3084             !list_is_first(&rule->list, &ctx->chain->rules) &&
3085             !list_is_last(&rule->list, &ctx->chain->rules)) {
3086                 prule = list_prev_entry(rule, list);
3087                 handle = prule->handle;
3088         }
3089         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3090                 flags |= NLM_F_APPEND;
3091         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3092                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3093
3094         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3095                                        event, flags, ctx->family, ctx->table,
3096                                        ctx->chain, rule, handle);
3097         if (err < 0) {
3098                 kfree_skb(skb);
3099                 goto err;
3100         }
3101
3102         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3103         return;
3104 err:
3105         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3106 }
3107
3108 struct nft_rule_dump_ctx {
3109         char *table;
3110         char *chain;
3111 };
3112
3113 static int __nf_tables_dump_rules(struct sk_buff *skb,
3114                                   unsigned int *idx,
3115                                   struct netlink_callback *cb,
3116                                   const struct nft_table *table,
3117                                   const struct nft_chain *chain)
3118 {
3119         struct net *net = sock_net(skb->sk);
3120         const struct nft_rule *rule, *prule;
3121         unsigned int s_idx = cb->args[0];
3122         u64 handle;
3123
3124         prule = NULL;
3125         list_for_each_entry_rcu(rule, &chain->rules, list) {
3126                 if (!nft_is_active(net, rule))
3127                         goto cont_skip;
3128                 if (*idx < s_idx)
3129                         goto cont;
3130                 if (*idx > s_idx) {
3131                         memset(&cb->args[1], 0,
3132                                         sizeof(cb->args) - sizeof(cb->args[0]));
3133                 }
3134                 if (prule)
3135                         handle = prule->handle;
3136                 else
3137                         handle = 0;
3138
3139                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3140                                         cb->nlh->nlmsg_seq,
3141                                         NFT_MSG_NEWRULE,
3142                                         NLM_F_MULTI | NLM_F_APPEND,
3143                                         table->family,
3144                                         table, chain, rule, handle) < 0)
3145                         return 1;
3146
3147                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3148 cont:
3149                 prule = rule;
3150 cont_skip:
3151                 (*idx)++;
3152         }
3153         return 0;
3154 }
3155
3156 static int nf_tables_dump_rules(struct sk_buff *skb,
3157                                 struct netlink_callback *cb)
3158 {
3159         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3160         const struct nft_rule_dump_ctx *ctx = cb->data;
3161         struct nft_table *table;
3162         const struct nft_chain *chain;
3163         unsigned int idx = 0;
3164         struct net *net = sock_net(skb->sk);
3165         int family = nfmsg->nfgen_family;
3166         struct nftables_pernet *nft_net;
3167
3168         rcu_read_lock();
3169         nft_net = nft_pernet(net);
3170         cb->seq = READ_ONCE(nft_net->base_seq);
3171
3172         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3173                 if (family != NFPROTO_UNSPEC && family != table->family)
3174                         continue;
3175
3176                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
3177                         continue;
3178
3179                 if (ctx && ctx->table && ctx->chain) {
3180                         struct rhlist_head *list, *tmp;
3181
3182                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3183                                                nft_chain_ht_params);
3184                         if (!list)
3185                                 goto done;
3186
3187                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3188                                 if (!nft_is_active(net, chain))
3189                                         continue;
3190                                 __nf_tables_dump_rules(skb, &idx,
3191                                                        cb, table, chain);
3192                                 break;
3193                         }
3194                         goto done;
3195                 }
3196
3197                 list_for_each_entry_rcu(chain, &table->chains, list) {
3198                         if (__nf_tables_dump_rules(skb, &idx, cb, table, chain))
3199                                 goto done;
3200                 }
3201
3202                 if (ctx && ctx->table)
3203                         break;
3204         }
3205 done:
3206         rcu_read_unlock();
3207
3208         cb->args[0] = idx;
3209         return skb->len;
3210 }
3211
3212 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3213 {
3214         const struct nlattr * const *nla = cb->data;
3215         struct nft_rule_dump_ctx *ctx = NULL;
3216
3217         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
3218                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
3219                 if (!ctx)
3220                         return -ENOMEM;
3221
3222                 if (nla[NFTA_RULE_TABLE]) {
3223                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
3224                                                         GFP_ATOMIC);
3225                         if (!ctx->table) {
3226                                 kfree(ctx);
3227                                 return -ENOMEM;
3228                         }
3229                 }
3230                 if (nla[NFTA_RULE_CHAIN]) {
3231                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
3232                                                 GFP_ATOMIC);
3233                         if (!ctx->chain) {
3234                                 kfree(ctx->table);
3235                                 kfree(ctx);
3236                                 return -ENOMEM;
3237                         }
3238                 }
3239         }
3240
3241         cb->data = ctx;
3242         return 0;
3243 }
3244
3245 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3246 {
3247         struct nft_rule_dump_ctx *ctx = cb->data;
3248
3249         if (ctx) {
3250                 kfree(ctx->table);
3251                 kfree(ctx->chain);
3252                 kfree(ctx);
3253         }
3254         return 0;
3255 }
3256
3257 /* called with rcu_read_lock held */
3258 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3259                              const struct nlattr * const nla[])
3260 {
3261         struct netlink_ext_ack *extack = info->extack;
3262         u8 genmask = nft_genmask_cur(info->net);
3263         u8 family = info->nfmsg->nfgen_family;
3264         const struct nft_chain *chain;
3265         const struct nft_rule *rule;
3266         struct net *net = info->net;
3267         struct nft_table *table;
3268         struct sk_buff *skb2;
3269         int err;
3270
3271         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3272                 struct netlink_dump_control c = {
3273                         .start= nf_tables_dump_rules_start,
3274                         .dump = nf_tables_dump_rules,
3275                         .done = nf_tables_dump_rules_done,
3276                         .module = THIS_MODULE,
3277                         .data = (void *)nla,
3278                 };
3279
3280                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3281         }
3282
3283         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3284         if (IS_ERR(table)) {
3285                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3286                 return PTR_ERR(table);
3287         }
3288
3289         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3290         if (IS_ERR(chain)) {
3291                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3292                 return PTR_ERR(chain);
3293         }
3294
3295         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3296         if (IS_ERR(rule)) {
3297                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3298                 return PTR_ERR(rule);
3299         }
3300
3301         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3302         if (!skb2)
3303                 return -ENOMEM;
3304
3305         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
3306                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3307                                        family, table, chain, rule, 0);
3308         if (err < 0)
3309                 goto err_fill_rule_info;
3310
3311         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
3312
3313 err_fill_rule_info:
3314         kfree_skb(skb2);
3315         return err;
3316 }
3317
3318 static void nf_tables_rule_destroy(const struct nft_ctx *ctx,
3319                                    struct nft_rule *rule)
3320 {
3321         struct nft_expr *expr, *next;
3322
3323         /*
3324          * Careful: some expressions might not be initialized in case this
3325          * is called on error from nf_tables_newrule().
3326          */
3327         expr = nft_expr_first(rule);
3328         while (nft_expr_more(rule, expr)) {
3329                 next = nft_expr_next(expr);
3330                 nf_tables_expr_destroy(ctx, expr);
3331                 expr = next;
3332         }
3333         kfree(rule);
3334 }
3335
3336 void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3337 {
3338         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3339         nf_tables_rule_destroy(ctx, rule);
3340 }
3341
3342 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3343 {
3344         struct nft_expr *expr, *last;
3345         const struct nft_data *data;
3346         struct nft_rule *rule;
3347         int err;
3348
3349         if (ctx->level == NFT_JUMP_STACK_SIZE)
3350                 return -EMLINK;
3351
3352         list_for_each_entry(rule, &chain->rules, list) {
3353                 if (!nft_is_active_next(ctx->net, rule))
3354                         continue;
3355
3356                 nft_rule_for_each_expr(expr, last, rule) {
3357                         if (!expr->ops->validate)
3358                                 continue;
3359
3360                         err = expr->ops->validate(ctx, expr, &data);
3361                         if (err < 0)
3362                                 return err;
3363                 }
3364
3365                 cond_resched();
3366         }
3367
3368         return 0;
3369 }
3370 EXPORT_SYMBOL_GPL(nft_chain_validate);
3371
3372 static int nft_table_validate(struct net *net, const struct nft_table *table)
3373 {
3374         struct nft_chain *chain;
3375         struct nft_ctx ctx = {
3376                 .net    = net,
3377                 .family = table->family,
3378         };
3379         int err;
3380
3381         list_for_each_entry(chain, &table->chains, list) {
3382                 if (!nft_is_base_chain(chain))
3383                         continue;
3384
3385                 ctx.chain = chain;
3386                 err = nft_chain_validate(&ctx, chain);
3387                 if (err < 0)
3388                         return err;
3389         }
3390
3391         return 0;
3392 }
3393
3394 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3395                                              const struct nft_chain *chain,
3396                                              const struct nlattr *nla);
3397
3398 #define NFT_RULE_MAXEXPRS       128
3399
3400 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3401                              const struct nlattr * const nla[])
3402 {
3403         struct nftables_pernet *nft_net = nft_pernet(info->net);
3404         struct netlink_ext_ack *extack = info->extack;
3405         unsigned int size, i, n, ulen = 0, usize = 0;
3406         u8 genmask = nft_genmask_next(info->net);
3407         struct nft_rule *rule, *old_rule = NULL;
3408         struct nft_expr_info *expr_info = NULL;
3409         u8 family = info->nfmsg->nfgen_family;
3410         struct nft_flow_rule *flow = NULL;
3411         struct net *net = info->net;
3412         struct nft_userdata *udata;
3413         struct nft_table *table;
3414         struct nft_chain *chain;
3415         struct nft_trans *trans;
3416         u64 handle, pos_handle;
3417         struct nft_expr *expr;
3418         struct nft_ctx ctx;
3419         struct nlattr *tmp;
3420         int err, rem;
3421
3422         lockdep_assert_held(&nft_net->commit_mutex);
3423
3424         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3425                                  NETLINK_CB(skb).portid);
3426         if (IS_ERR(table)) {
3427                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3428                 return PTR_ERR(table);
3429         }
3430
3431         if (nla[NFTA_RULE_CHAIN]) {
3432                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3433                                          genmask);
3434                 if (IS_ERR(chain)) {
3435                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3436                         return PTR_ERR(chain);
3437                 }
3438                 if (nft_chain_is_bound(chain))
3439                         return -EOPNOTSUPP;
3440
3441         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3442                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID]);
3443                 if (IS_ERR(chain)) {
3444                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3445                         return PTR_ERR(chain);
3446                 }
3447         } else {
3448                 return -EINVAL;
3449         }
3450
3451         if (nla[NFTA_RULE_HANDLE]) {
3452                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3453                 rule = __nft_rule_lookup(chain, handle);
3454                 if (IS_ERR(rule)) {
3455                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3456                         return PTR_ERR(rule);
3457                 }
3458
3459                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3460                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3461                         return -EEXIST;
3462                 }
3463                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3464                         old_rule = rule;
3465                 else
3466                         return -EOPNOTSUPP;
3467         } else {
3468                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3469                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3470                         return -EINVAL;
3471                 handle = nf_tables_alloc_handle(table);
3472
3473                 if (chain->use == UINT_MAX)
3474                         return -EOVERFLOW;
3475
3476                 if (nla[NFTA_RULE_POSITION]) {
3477                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3478                         old_rule = __nft_rule_lookup(chain, pos_handle);
3479                         if (IS_ERR(old_rule)) {
3480                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3481                                 return PTR_ERR(old_rule);
3482                         }
3483                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3484                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
3485                         if (IS_ERR(old_rule)) {
3486                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3487                                 return PTR_ERR(old_rule);
3488                         }
3489                 }
3490         }
3491
3492         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3493
3494         n = 0;
3495         size = 0;
3496         if (nla[NFTA_RULE_EXPRESSIONS]) {
3497                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3498                                            sizeof(struct nft_expr_info),
3499                                            GFP_KERNEL);
3500                 if (!expr_info)
3501                         return -ENOMEM;
3502
3503                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3504                         err = -EINVAL;
3505                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3506                                 goto err_release_expr;
3507                         if (n == NFT_RULE_MAXEXPRS)
3508                                 goto err_release_expr;
3509                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3510                         if (err < 0) {
3511                                 NL_SET_BAD_ATTR(extack, tmp);
3512                                 goto err_release_expr;
3513                         }
3514                         size += expr_info[n].ops->size;
3515                         n++;
3516                 }
3517         }
3518         /* Check for overflow of dlen field */
3519         err = -EFBIG;
3520         if (size >= 1 << 12)
3521                 goto err_release_expr;
3522
3523         if (nla[NFTA_RULE_USERDATA]) {
3524                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3525                 if (ulen > 0)
3526                         usize = sizeof(struct nft_userdata) + ulen;
3527         }
3528
3529         err = -ENOMEM;
3530         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
3531         if (rule == NULL)
3532                 goto err_release_expr;
3533
3534         nft_activate_next(net, rule);
3535
3536         rule->handle = handle;
3537         rule->dlen   = size;
3538         rule->udata  = ulen ? 1 : 0;
3539
3540         if (ulen) {
3541                 udata = nft_userdata(rule);
3542                 udata->len = ulen - 1;
3543                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3544         }
3545
3546         expr = nft_expr_first(rule);
3547         for (i = 0; i < n; i++) {
3548                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
3549                 if (err < 0) {
3550                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
3551                         goto err_release_rule;
3552                 }
3553
3554                 if (expr_info[i].ops->validate)
3555                         nft_validate_state_update(net, NFT_VALIDATE_NEED);
3556
3557                 expr_info[i].ops = NULL;
3558                 expr = nft_expr_next(expr);
3559         }
3560
3561         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3562                 flow = nft_flow_rule_create(net, rule);
3563                 if (IS_ERR(flow)) {
3564                         err = PTR_ERR(flow);
3565                         goto err_release_rule;
3566                 }
3567         }
3568
3569         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
3570                 err = nft_delrule(&ctx, old_rule);
3571                 if (err < 0)
3572                         goto err_destroy_flow_rule;
3573
3574                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3575                 if (trans == NULL) {
3576                         err = -ENOMEM;
3577                         goto err_destroy_flow_rule;
3578                 }
3579                 list_add_tail_rcu(&rule->list, &old_rule->list);
3580         } else {
3581                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3582                 if (!trans) {
3583                         err = -ENOMEM;
3584                         goto err_destroy_flow_rule;
3585                 }
3586
3587                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
3588                         if (old_rule)
3589                                 list_add_rcu(&rule->list, &old_rule->list);
3590                         else
3591                                 list_add_tail_rcu(&rule->list, &chain->rules);
3592                  } else {
3593                         if (old_rule)
3594                                 list_add_tail_rcu(&rule->list, &old_rule->list);
3595                         else
3596                                 list_add_rcu(&rule->list, &chain->rules);
3597                 }
3598         }
3599         kvfree(expr_info);
3600         chain->use++;
3601
3602         if (flow)
3603                 nft_trans_flow_rule(trans) = flow;
3604
3605         if (nft_net->validate_state == NFT_VALIDATE_DO)
3606                 return nft_table_validate(net, table);
3607
3608         return 0;
3609
3610 err_destroy_flow_rule:
3611         if (flow)
3612                 nft_flow_rule_destroy(flow);
3613 err_release_rule:
3614         nf_tables_rule_release(&ctx, rule);
3615 err_release_expr:
3616         for (i = 0; i < n; i++) {
3617                 if (expr_info[i].ops) {
3618                         module_put(expr_info[i].ops->type->owner);
3619                         if (expr_info[i].ops->type->release_ops)
3620                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
3621                 }
3622         }
3623         kvfree(expr_info);
3624
3625         return err;
3626 }
3627
3628 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3629                                              const struct nft_chain *chain,
3630                                              const struct nlattr *nla)
3631 {
3632         struct nftables_pernet *nft_net = nft_pernet(net);
3633         u32 id = ntohl(nla_get_be32(nla));
3634         struct nft_trans *trans;
3635
3636         list_for_each_entry(trans, &nft_net->commit_list, list) {
3637                 struct nft_rule *rule = nft_trans_rule(trans);
3638
3639                 if (trans->msg_type == NFT_MSG_NEWRULE &&
3640                     trans->ctx.chain == chain &&
3641                     id == nft_trans_rule_id(trans))
3642                         return rule;
3643         }
3644         return ERR_PTR(-ENOENT);
3645 }
3646
3647 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
3648                              const struct nlattr * const nla[])
3649 {
3650         struct netlink_ext_ack *extack = info->extack;
3651         u8 genmask = nft_genmask_next(info->net);
3652         u8 family = info->nfmsg->nfgen_family;
3653         struct nft_chain *chain = NULL;
3654         struct net *net = info->net;
3655         struct nft_table *table;
3656         struct nft_rule *rule;
3657         struct nft_ctx ctx;
3658         int err = 0;
3659
3660         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3661                                  NETLINK_CB(skb).portid);
3662         if (IS_ERR(table)) {
3663                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3664                 return PTR_ERR(table);
3665         }
3666
3667         if (nla[NFTA_RULE_CHAIN]) {
3668                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3669                                          genmask);
3670                 if (IS_ERR(chain)) {
3671                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3672                         return PTR_ERR(chain);
3673                 }
3674                 if (nft_chain_is_bound(chain))
3675                         return -EOPNOTSUPP;
3676         }
3677
3678         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3679
3680         if (chain) {
3681                 if (nla[NFTA_RULE_HANDLE]) {
3682                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3683                         if (IS_ERR(rule)) {
3684                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3685                                 return PTR_ERR(rule);
3686                         }
3687
3688                         err = nft_delrule(&ctx, rule);
3689                 } else if (nla[NFTA_RULE_ID]) {
3690                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
3691                         if (IS_ERR(rule)) {
3692                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
3693                                 return PTR_ERR(rule);
3694                         }
3695
3696                         err = nft_delrule(&ctx, rule);
3697                 } else {
3698                         err = nft_delrule_by_chain(&ctx);
3699                 }
3700         } else {
3701                 list_for_each_entry(chain, &table->chains, list) {
3702                         if (!nft_is_active_next(net, chain))
3703                                 continue;
3704
3705                         ctx.chain = chain;
3706                         err = nft_delrule_by_chain(&ctx);
3707                         if (err < 0)
3708                                 break;
3709                 }
3710         }
3711
3712         return err;
3713 }
3714
3715 /*
3716  * Sets
3717  */
3718 static const struct nft_set_type *nft_set_types[] = {
3719         &nft_set_hash_fast_type,
3720         &nft_set_hash_type,
3721         &nft_set_rhash_type,
3722         &nft_set_bitmap_type,
3723         &nft_set_rbtree_type,
3724 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
3725         &nft_set_pipapo_avx2_type,
3726 #endif
3727         &nft_set_pipapo_type,
3728 };
3729
3730 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
3731                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
3732                                  NFT_SET_EVAL)
3733
3734 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
3735 {
3736         return (flags & type->features) == (flags & NFT_SET_FEATURES);
3737 }
3738
3739 /*
3740  * Select a set implementation based on the data characteristics and the
3741  * given policy. The total memory use might not be known if no size is
3742  * given, in that case the amount of memory per element is used.
3743  */
3744 static const struct nft_set_ops *
3745 nft_select_set_ops(const struct nft_ctx *ctx,
3746                    const struct nlattr * const nla[],
3747                    const struct nft_set_desc *desc)
3748 {
3749         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3750         const struct nft_set_ops *ops, *bops;
3751         struct nft_set_estimate est, best;
3752         const struct nft_set_type *type;
3753         u32 flags = 0;
3754         int i;
3755
3756         lockdep_assert_held(&nft_net->commit_mutex);
3757         lockdep_nfnl_nft_mutex_not_held();
3758
3759         if (nla[NFTA_SET_FLAGS] != NULL)
3760                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
3761
3762         bops        = NULL;
3763         best.size   = ~0;
3764         best.lookup = ~0;
3765         best.space  = ~0;
3766
3767         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
3768                 type = nft_set_types[i];
3769                 ops = &type->ops;
3770
3771                 if (!nft_set_ops_candidate(type, flags))
3772                         continue;
3773                 if (!ops->estimate(desc, flags, &est))
3774                         continue;
3775
3776                 switch (desc->policy) {
3777                 case NFT_SET_POL_PERFORMANCE:
3778                         if (est.lookup < best.lookup)
3779                                 break;
3780                         if (est.lookup == best.lookup &&
3781                             est.space < best.space)
3782                                 break;
3783                         continue;
3784                 case NFT_SET_POL_MEMORY:
3785                         if (!desc->size) {
3786                                 if (est.space < best.space)
3787                                         break;
3788                                 if (est.space == best.space &&
3789                                     est.lookup < best.lookup)
3790                                         break;
3791                         } else if (est.size < best.size || !bops) {
3792                                 break;
3793                         }
3794                         continue;
3795                 default:
3796                         break;
3797                 }
3798
3799                 bops = ops;
3800                 best = est;
3801         }
3802
3803         if (bops != NULL)
3804                 return bops;
3805
3806         return ERR_PTR(-EOPNOTSUPP);
3807 }
3808
3809 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
3810         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
3811                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
3812         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
3813                                             .len = NFT_SET_MAXNAMELEN - 1 },
3814         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
3815         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
3816         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
3817         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
3818         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
3819         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
3820         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
3821         [NFTA_SET_ID]                   = { .type = NLA_U32 },
3822         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
3823         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
3824         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
3825                                             .len  = NFT_USERDATA_MAXLEN },
3826         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
3827         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
3828         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
3829         [NFTA_SET_EXPRESSIONS]          = { .type = NLA_NESTED },
3830 };
3831
3832 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
3833         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
3834         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
3835 };
3836
3837 static struct nft_set *nft_set_lookup(const struct nft_table *table,
3838                                       const struct nlattr *nla, u8 genmask)
3839 {
3840         struct nft_set *set;
3841
3842         if (nla == NULL)
3843                 return ERR_PTR(-EINVAL);
3844
3845         list_for_each_entry_rcu(set, &table->sets, list) {
3846                 if (!nla_strcmp(nla, set->name) &&
3847                     nft_active_genmask(set, genmask))
3848                         return set;
3849         }
3850         return ERR_PTR(-ENOENT);
3851 }
3852
3853 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
3854                                                const struct nlattr *nla,
3855                                                u8 genmask)
3856 {
3857         struct nft_set *set;
3858
3859         list_for_each_entry(set, &table->sets, list) {
3860                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
3861                     nft_active_genmask(set, genmask))
3862                         return set;
3863         }
3864         return ERR_PTR(-ENOENT);
3865 }
3866
3867 static struct nft_set *nft_set_lookup_byid(const struct net *net,
3868                                            const struct nft_table *table,
3869                                            const struct nlattr *nla, u8 genmask)
3870 {
3871         struct nftables_pernet *nft_net = nft_pernet(net);
3872         u32 id = ntohl(nla_get_be32(nla));
3873         struct nft_trans *trans;
3874
3875         list_for_each_entry(trans, &nft_net->commit_list, list) {
3876                 if (trans->msg_type == NFT_MSG_NEWSET) {
3877                         struct nft_set *set = nft_trans_set(trans);
3878
3879                         if (id == nft_trans_set_id(trans) &&
3880                             set->table == table &&
3881                             nft_active_genmask(set, genmask))
3882                                 return set;
3883                 }
3884         }
3885         return ERR_PTR(-ENOENT);
3886 }
3887
3888 struct nft_set *nft_set_lookup_global(const struct net *net,
3889                                       const struct nft_table *table,
3890                                       const struct nlattr *nla_set_name,
3891                                       const struct nlattr *nla_set_id,
3892                                       u8 genmask)
3893 {
3894         struct nft_set *set;
3895
3896         set = nft_set_lookup(table, nla_set_name, genmask);
3897         if (IS_ERR(set)) {
3898                 if (!nla_set_id)
3899                         return set;
3900
3901                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
3902         }
3903         return set;
3904 }
3905 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
3906
3907 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
3908                                     const char *name)
3909 {
3910         const struct nft_set *i;
3911         const char *p;
3912         unsigned long *inuse;
3913         unsigned int n = 0, min = 0;
3914
3915         p = strchr(name, '%');
3916         if (p != NULL) {
3917                 if (p[1] != 'd' || strchr(p + 2, '%'))
3918                         return -EINVAL;
3919
3920                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
3921                 if (inuse == NULL)
3922                         return -ENOMEM;
3923 cont:
3924                 list_for_each_entry(i, &ctx->table->sets, list) {
3925                         int tmp;
3926
3927                         if (!nft_is_active_next(ctx->net, i))
3928                                 continue;
3929                         if (!sscanf(i->name, name, &tmp))
3930                                 continue;
3931                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
3932                                 continue;
3933
3934                         set_bit(tmp - min, inuse);
3935                 }
3936
3937                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
3938                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
3939                         min += BITS_PER_BYTE * PAGE_SIZE;
3940                         memset(inuse, 0, PAGE_SIZE);
3941                         goto cont;
3942                 }
3943                 free_page((unsigned long)inuse);
3944         }
3945
3946         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
3947         if (!set->name)
3948                 return -ENOMEM;
3949
3950         list_for_each_entry(i, &ctx->table->sets, list) {
3951                 if (!nft_is_active_next(ctx->net, i))
3952                         continue;
3953                 if (!strcmp(set->name, i->name)) {
3954                         kfree(set->name);
3955                         set->name = NULL;
3956                         return -ENFILE;
3957                 }
3958         }
3959         return 0;
3960 }
3961
3962 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
3963 {
3964         u64 ms = be64_to_cpu(nla_get_be64(nla));
3965         u64 max = (u64)(~((u64)0));
3966
3967         max = div_u64(max, NSEC_PER_MSEC);
3968         if (ms >= max)
3969                 return -ERANGE;
3970
3971         ms *= NSEC_PER_MSEC;
3972         *result = nsecs_to_jiffies64(ms);
3973         return 0;
3974 }
3975
3976 __be64 nf_jiffies64_to_msecs(u64 input)
3977 {
3978         return cpu_to_be64(jiffies64_to_msecs(input));
3979 }
3980
3981 static int nf_tables_fill_set_concat(struct sk_buff *skb,
3982                                      const struct nft_set *set)
3983 {
3984         struct nlattr *concat, *field;
3985         int i;
3986
3987         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
3988         if (!concat)
3989                 return -ENOMEM;
3990
3991         for (i = 0; i < set->field_count; i++) {
3992                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
3993                 if (!field)
3994                         return -ENOMEM;
3995
3996                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
3997                                  htonl(set->field_len[i])))
3998                         return -ENOMEM;
3999
4000                 nla_nest_end(skb, field);
4001         }
4002
4003         nla_nest_end(skb, concat);
4004
4005         return 0;
4006 }
4007
4008 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4009                               const struct nft_set *set, u16 event, u16 flags)
4010 {
4011         u64 timeout = READ_ONCE(set->timeout);
4012         u32 gc_int = READ_ONCE(set->gc_int);
4013         u32 portid = ctx->portid;
4014         struct nlmsghdr *nlh;
4015         struct nlattr *nest;
4016         u32 seq = ctx->seq;
4017         int i;
4018
4019         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4020         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4021                            NFNETLINK_V0, nft_base_seq(ctx->net));
4022         if (!nlh)
4023                 goto nla_put_failure;
4024
4025         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4026                 goto nla_put_failure;
4027         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4028                 goto nla_put_failure;
4029         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4030                          NFTA_SET_PAD))
4031                 goto nla_put_failure;
4032         if (set->flags != 0)
4033                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4034                         goto nla_put_failure;
4035
4036         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4037                 goto nla_put_failure;
4038         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4039                 goto nla_put_failure;
4040         if (set->flags & NFT_SET_MAP) {
4041                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4042                         goto nla_put_failure;
4043                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4044                         goto nla_put_failure;
4045         }
4046         if (set->flags & NFT_SET_OBJECT &&
4047             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4048                 goto nla_put_failure;
4049
4050         if (timeout &&
4051             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4052                          nf_jiffies64_to_msecs(timeout),
4053                          NFTA_SET_PAD))
4054                 goto nla_put_failure;
4055         if (gc_int &&
4056             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4057                 goto nla_put_failure;
4058
4059         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4060                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4061                         goto nla_put_failure;
4062         }
4063
4064         if (set->udata &&
4065             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4066                 goto nla_put_failure;
4067
4068         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4069         if (!nest)
4070                 goto nla_put_failure;
4071         if (set->size &&
4072             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4073                 goto nla_put_failure;
4074
4075         if (set->field_count > 1 &&
4076             nf_tables_fill_set_concat(skb, set))
4077                 goto nla_put_failure;
4078
4079         nla_nest_end(skb, nest);
4080
4081         if (set->num_exprs == 1) {
4082                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4083                 if (nf_tables_fill_expr_info(skb, set->exprs[0]) < 0)
4084                         goto nla_put_failure;
4085
4086                 nla_nest_end(skb, nest);
4087         } else if (set->num_exprs > 1) {
4088                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4089                 if (nest == NULL)
4090                         goto nla_put_failure;
4091
4092                 for (i = 0; i < set->num_exprs; i++) {
4093                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4094                                           set->exprs[i]) < 0)
4095                                 goto nla_put_failure;
4096                 }
4097                 nla_nest_end(skb, nest);
4098         }
4099
4100         nlmsg_end(skb, nlh);
4101         return 0;
4102
4103 nla_put_failure:
4104         nlmsg_trim(skb, nlh);
4105         return -1;
4106 }
4107
4108 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4109                                  const struct nft_set *set, int event,
4110                                  gfp_t gfp_flags)
4111 {
4112         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4113         u32 portid = ctx->portid;
4114         struct sk_buff *skb;
4115         u16 flags = 0;
4116         int err;
4117
4118         if (!ctx->report &&
4119             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4120                 return;
4121
4122         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4123         if (skb == NULL)
4124                 goto err;
4125
4126         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4127                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4128
4129         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4130         if (err < 0) {
4131                 kfree_skb(skb);
4132                 goto err;
4133         }
4134
4135         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4136         return;
4137 err:
4138         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4139 }
4140
4141 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4142 {
4143         const struct nft_set *set;
4144         unsigned int idx, s_idx = cb->args[0];
4145         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4146         struct net *net = sock_net(skb->sk);
4147         struct nft_ctx *ctx = cb->data, ctx_set;
4148         struct nftables_pernet *nft_net;
4149
4150         if (cb->args[1])
4151                 return skb->len;
4152
4153         rcu_read_lock();
4154         nft_net = nft_pernet(net);
4155         cb->seq = READ_ONCE(nft_net->base_seq);
4156
4157         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4158                 if (ctx->family != NFPROTO_UNSPEC &&
4159                     ctx->family != table->family)
4160                         continue;
4161
4162                 if (ctx->table && ctx->table != table)
4163                         continue;
4164
4165                 if (cur_table) {
4166                         if (cur_table != table)
4167                                 continue;
4168
4169                         cur_table = NULL;
4170                 }
4171                 idx = 0;
4172                 list_for_each_entry_rcu(set, &table->sets, list) {
4173                         if (idx < s_idx)
4174                                 goto cont;
4175                         if (!nft_is_active(net, set))
4176                                 goto cont;
4177
4178                         ctx_set = *ctx;
4179                         ctx_set.table = table;
4180                         ctx_set.family = table->family;
4181
4182                         if (nf_tables_fill_set(skb, &ctx_set, set,
4183                                                NFT_MSG_NEWSET,
4184                                                NLM_F_MULTI) < 0) {
4185                                 cb->args[0] = idx;
4186                                 cb->args[2] = (unsigned long) table;
4187                                 goto done;
4188                         }
4189                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4190 cont:
4191                         idx++;
4192                 }
4193                 if (s_idx)
4194                         s_idx = 0;
4195         }
4196         cb->args[1] = 1;
4197 done:
4198         rcu_read_unlock();
4199         return skb->len;
4200 }
4201
4202 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4203 {
4204         struct nft_ctx *ctx_dump = NULL;
4205
4206         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4207         if (ctx_dump == NULL)
4208                 return -ENOMEM;
4209
4210         cb->data = ctx_dump;
4211         return 0;
4212 }
4213
4214 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4215 {
4216         kfree(cb->data);
4217         return 0;
4218 }
4219
4220 /* called with rcu_read_lock held */
4221 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4222                             const struct nlattr * const nla[])
4223 {
4224         struct netlink_ext_ack *extack = info->extack;
4225         u8 genmask = nft_genmask_cur(info->net);
4226         u8 family = info->nfmsg->nfgen_family;
4227         struct nft_table *table = NULL;
4228         struct net *net = info->net;
4229         const struct nft_set *set;
4230         struct sk_buff *skb2;
4231         struct nft_ctx ctx;
4232         int err;
4233
4234         if (nla[NFTA_SET_TABLE]) {
4235                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4236                                          genmask, 0);
4237                 if (IS_ERR(table)) {
4238                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4239                         return PTR_ERR(table);
4240                 }
4241         }
4242
4243         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4244
4245         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4246                 struct netlink_dump_control c = {
4247                         .start = nf_tables_dump_sets_start,
4248                         .dump = nf_tables_dump_sets,
4249                         .done = nf_tables_dump_sets_done,
4250                         .data = &ctx,
4251                         .module = THIS_MODULE,
4252                 };
4253
4254                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4255         }
4256
4257         /* Only accept unspec with dump */
4258         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4259                 return -EAFNOSUPPORT;
4260         if (!nla[NFTA_SET_TABLE])
4261                 return -EINVAL;
4262
4263         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4264         if (IS_ERR(set))
4265                 return PTR_ERR(set);
4266
4267         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4268         if (skb2 == NULL)
4269                 return -ENOMEM;
4270
4271         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4272         if (err < 0)
4273                 goto err_fill_set_info;
4274
4275         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4276
4277 err_fill_set_info:
4278         kfree_skb(skb2);
4279         return err;
4280 }
4281
4282 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4283         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4284 };
4285
4286 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4287                                      struct nft_set_desc *desc)
4288 {
4289         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4290         u32 len;
4291         int err;
4292
4293         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4294                 return -E2BIG;
4295
4296         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4297                                           nft_concat_policy, NULL);
4298         if (err < 0)
4299                 return err;
4300
4301         if (!tb[NFTA_SET_FIELD_LEN])
4302                 return -EINVAL;
4303
4304         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4305         if (!len || len > U8_MAX)
4306                 return -EINVAL;
4307
4308         desc->field_len[desc->field_count++] = len;
4309
4310         return 0;
4311 }
4312
4313 static int nft_set_desc_concat(struct nft_set_desc *desc,
4314                                const struct nlattr *nla)
4315 {
4316         struct nlattr *attr;
4317         u32 num_regs = 0;
4318         int rem, err, i;
4319
4320         nla_for_each_nested(attr, nla, rem) {
4321                 if (nla_type(attr) != NFTA_LIST_ELEM)
4322                         return -EINVAL;
4323
4324                 err = nft_set_desc_concat_parse(attr, desc);
4325                 if (err < 0)
4326                         return err;
4327         }
4328
4329         for (i = 0; i < desc->field_count; i++)
4330                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4331
4332         if (num_regs > NFT_REG32_COUNT)
4333                 return -E2BIG;
4334
4335         return 0;
4336 }
4337
4338 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4339                                     const struct nlattr *nla)
4340 {
4341         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4342         int err;
4343
4344         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4345                                           nft_set_desc_policy, NULL);
4346         if (err < 0)
4347                 return err;
4348
4349         if (da[NFTA_SET_DESC_SIZE] != NULL)
4350                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4351         if (da[NFTA_SET_DESC_CONCAT])
4352                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4353
4354         return err;
4355 }
4356
4357 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4358                               const struct nlattr * const *nla,
4359                               struct nft_expr **exprs, int *num_exprs,
4360                               u32 flags)
4361 {
4362         struct nft_expr *expr;
4363         int err, i;
4364
4365         if (nla[NFTA_SET_EXPR]) {
4366                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4367                 if (IS_ERR(expr)) {
4368                         err = PTR_ERR(expr);
4369                         goto err_set_expr_alloc;
4370                 }
4371                 exprs[0] = expr;
4372                 (*num_exprs)++;
4373         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4374                 struct nlattr *tmp;
4375                 int left;
4376
4377                 if (!(flags & NFT_SET_EXPR)) {
4378                         err = -EINVAL;
4379                         goto err_set_expr_alloc;
4380                 }
4381                 i = 0;
4382                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4383                         if (i == NFT_SET_EXPR_MAX) {
4384                                 err = -E2BIG;
4385                                 goto err_set_expr_alloc;
4386                         }
4387                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4388                                 err = -EINVAL;
4389                                 goto err_set_expr_alloc;
4390                         }
4391                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4392                         if (IS_ERR(expr)) {
4393                                 err = PTR_ERR(expr);
4394                                 goto err_set_expr_alloc;
4395                         }
4396                         exprs[i++] = expr;
4397                         (*num_exprs)++;
4398                 }
4399         }
4400
4401         return 0;
4402
4403 err_set_expr_alloc:
4404         for (i = 0; i < *num_exprs; i++)
4405                 nft_expr_destroy(ctx, exprs[i]);
4406
4407         return err;
4408 }
4409
4410 static bool nft_set_is_same(const struct nft_set *set,
4411                             const struct nft_set_desc *desc,
4412                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4413 {
4414         int i;
4415
4416         if (set->ktype != desc->ktype ||
4417             set->dtype != desc->dtype ||
4418             set->flags != flags ||
4419             set->klen != desc->klen ||
4420             set->dlen != desc->dlen ||
4421             set->field_count != desc->field_count ||
4422             set->num_exprs != num_exprs)
4423                 return false;
4424
4425         for (i = 0; i < desc->field_count; i++) {
4426                 if (set->field_len[i] != desc->field_len[i])
4427                         return false;
4428         }
4429
4430         for (i = 0; i < num_exprs; i++) {
4431                 if (set->exprs[i]->ops != exprs[i]->ops)
4432                         return false;
4433         }
4434
4435         return true;
4436 }
4437
4438 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4439                             const struct nlattr * const nla[])
4440 {
4441         struct netlink_ext_ack *extack = info->extack;
4442         u8 genmask = nft_genmask_next(info->net);
4443         u8 family = info->nfmsg->nfgen_family;
4444         const struct nft_set_ops *ops;
4445         struct net *net = info->net;
4446         struct nft_set_desc desc;
4447         struct nft_table *table;
4448         unsigned char *udata;
4449         struct nft_set *set;
4450         struct nft_ctx ctx;
4451         size_t alloc_size;
4452         int num_exprs = 0;
4453         char *name;
4454         int err, i;
4455         u16 udlen;
4456         u32 flags;
4457         u64 size;
4458
4459         if (nla[NFTA_SET_TABLE] == NULL ||
4460             nla[NFTA_SET_NAME] == NULL ||
4461             nla[NFTA_SET_KEY_LEN] == NULL ||
4462             nla[NFTA_SET_ID] == NULL)
4463                 return -EINVAL;
4464
4465         memset(&desc, 0, sizeof(desc));
4466
4467         desc.ktype = NFT_DATA_VALUE;
4468         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4469                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4470                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4471                         return -EINVAL;
4472         }
4473
4474         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4475         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4476                 return -EINVAL;
4477
4478         flags = 0;
4479         if (nla[NFTA_SET_FLAGS] != NULL) {
4480                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4481                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4482                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4483                               NFT_SET_MAP | NFT_SET_EVAL |
4484                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4485                         return -EOPNOTSUPP;
4486                 /* Only one of these operations is supported */
4487                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4488                              (NFT_SET_MAP | NFT_SET_OBJECT))
4489                         return -EOPNOTSUPP;
4490                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4491                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4492                         return -EOPNOTSUPP;
4493         }
4494
4495         desc.dtype = 0;
4496         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4497                 if (!(flags & NFT_SET_MAP))
4498                         return -EINVAL;
4499
4500                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4501                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4502                     desc.dtype != NFT_DATA_VERDICT)
4503                         return -EINVAL;
4504
4505                 if (desc.dtype != NFT_DATA_VERDICT) {
4506                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4507                                 return -EINVAL;
4508                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4509                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4510                                 return -EINVAL;
4511                 } else
4512                         desc.dlen = sizeof(struct nft_verdict);
4513         } else if (flags & NFT_SET_MAP)
4514                 return -EINVAL;
4515
4516         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4517                 if (!(flags & NFT_SET_OBJECT))
4518                         return -EINVAL;
4519
4520                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4521                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
4522                     desc.objtype > NFT_OBJECT_MAX)
4523                         return -EOPNOTSUPP;
4524         } else if (flags & NFT_SET_OBJECT)
4525                 return -EINVAL;
4526         else
4527                 desc.objtype = NFT_OBJECT_UNSPEC;
4528
4529         desc.timeout = 0;
4530         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4531                 if (!(flags & NFT_SET_TIMEOUT))
4532                         return -EINVAL;
4533
4534                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
4535                 if (err)
4536                         return err;
4537         }
4538         desc.gc_int = 0;
4539         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4540                 if (!(flags & NFT_SET_TIMEOUT))
4541                         return -EINVAL;
4542                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4543         }
4544
4545         desc.policy = NFT_SET_POL_PERFORMANCE;
4546         if (nla[NFTA_SET_POLICY] != NULL)
4547                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
4548
4549         if (nla[NFTA_SET_DESC] != NULL) {
4550                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
4551                 if (err < 0)
4552                         return err;
4553
4554                 if (desc.field_count > 1 && !(flags & NFT_SET_CONCAT))
4555                         return -EINVAL;
4556         } else if (flags & NFT_SET_CONCAT) {
4557                 return -EINVAL;
4558         }
4559
4560         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
4561                 desc.expr = true;
4562
4563         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
4564                                  NETLINK_CB(skb).portid);
4565         if (IS_ERR(table)) {
4566                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4567                 return PTR_ERR(table);
4568         }
4569
4570         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4571
4572         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4573         if (IS_ERR(set)) {
4574                 if (PTR_ERR(set) != -ENOENT) {
4575                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4576                         return PTR_ERR(set);
4577                 }
4578         } else {
4579                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
4580
4581                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4582                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4583                         return -EEXIST;
4584                 }
4585                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4586                         return -EOPNOTSUPP;
4587
4588                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
4589                 if (err < 0)
4590                         return err;
4591
4592                 err = 0;
4593                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
4594                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4595                         err = -EEXIST;
4596                 }
4597
4598                 for (i = 0; i < num_exprs; i++)
4599                         nft_expr_destroy(&ctx, exprs[i]);
4600
4601                 if (err < 0)
4602                         return err;
4603
4604                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
4605         }
4606
4607         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
4608                 return -ENOENT;
4609
4610         ops = nft_select_set_ops(&ctx, nla, &desc);
4611         if (IS_ERR(ops))
4612                 return PTR_ERR(ops);
4613
4614         udlen = 0;
4615         if (nla[NFTA_SET_USERDATA])
4616                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
4617
4618         size = 0;
4619         if (ops->privsize != NULL)
4620                 size = ops->privsize(nla, &desc);
4621         alloc_size = sizeof(*set) + size + udlen;
4622         if (alloc_size < size || alloc_size > INT_MAX)
4623                 return -ENOMEM;
4624         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
4625         if (!set)
4626                 return -ENOMEM;
4627
4628         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
4629         if (!name) {
4630                 err = -ENOMEM;
4631                 goto err_set_name;
4632         }
4633
4634         err = nf_tables_set_alloc_name(&ctx, set, name);
4635         kfree(name);
4636         if (err < 0)
4637                 goto err_set_name;
4638
4639         udata = NULL;
4640         if (udlen) {
4641                 udata = set->data + size;
4642                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
4643         }
4644
4645         INIT_LIST_HEAD(&set->bindings);
4646         INIT_LIST_HEAD(&set->catchall_list);
4647         set->table = table;
4648         write_pnet(&set->net, net);
4649         set->ops = ops;
4650         set->ktype = desc.ktype;
4651         set->klen = desc.klen;
4652         set->dtype = desc.dtype;
4653         set->objtype = desc.objtype;
4654         set->dlen = desc.dlen;
4655         set->flags = flags;
4656         set->size = desc.size;
4657         set->policy = desc.policy;
4658         set->udlen = udlen;
4659         set->udata = udata;
4660         set->timeout = desc.timeout;
4661         set->gc_int = desc.gc_int;
4662
4663         set->field_count = desc.field_count;
4664         for (i = 0; i < desc.field_count; i++)
4665                 set->field_len[i] = desc.field_len[i];
4666
4667         err = ops->init(set, &desc, nla);
4668         if (err < 0)
4669                 goto err_set_init;
4670
4671         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
4672         if (err < 0)
4673                 goto err_set_destroy;
4674
4675         set->num_exprs = num_exprs;
4676         set->handle = nf_tables_alloc_handle(table);
4677
4678         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
4679         if (err < 0)
4680                 goto err_set_expr_alloc;
4681
4682         list_add_tail_rcu(&set->list, &table->sets);
4683         table->use++;
4684         return 0;
4685
4686 err_set_expr_alloc:
4687         for (i = 0; i < set->num_exprs; i++)
4688                 nft_expr_destroy(&ctx, set->exprs[i]);
4689 err_set_destroy:
4690         ops->destroy(set);
4691 err_set_init:
4692         kfree(set->name);
4693 err_set_name:
4694         kvfree(set);
4695         return err;
4696 }
4697
4698 struct nft_set_elem_catchall {
4699         struct list_head        list;
4700         struct rcu_head         rcu;
4701         void                    *elem;
4702 };
4703
4704 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
4705                                      struct nft_set *set)
4706 {
4707         struct nft_set_elem_catchall *next, *catchall;
4708
4709         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
4710                 list_del_rcu(&catchall->list);
4711                 nft_set_elem_destroy(set, catchall->elem, true);
4712                 kfree_rcu(catchall, rcu);
4713         }
4714 }
4715
4716 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
4717 {
4718         int i;
4719
4720         if (WARN_ON(set->use > 0))
4721                 return;
4722
4723         for (i = 0; i < set->num_exprs; i++)
4724                 nft_expr_destroy(ctx, set->exprs[i]);
4725
4726         set->ops->destroy(set);
4727         nft_set_catchall_destroy(ctx, set);
4728         kfree(set->name);
4729         kvfree(set);
4730 }
4731
4732 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
4733                             const struct nlattr * const nla[])
4734 {
4735         struct netlink_ext_ack *extack = info->extack;
4736         u8 genmask = nft_genmask_next(info->net);
4737         u8 family = info->nfmsg->nfgen_family;
4738         struct net *net = info->net;
4739         const struct nlattr *attr;
4740         struct nft_table *table;
4741         struct nft_set *set;
4742         struct nft_ctx ctx;
4743
4744         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4745                 return -EAFNOSUPPORT;
4746
4747         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4748                                  genmask, NETLINK_CB(skb).portid);
4749         if (IS_ERR(table)) {
4750                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4751                 return PTR_ERR(table);
4752         }
4753
4754         if (nla[NFTA_SET_HANDLE]) {
4755                 attr = nla[NFTA_SET_HANDLE];
4756                 set = nft_set_lookup_byhandle(table, attr, genmask);
4757         } else {
4758                 attr = nla[NFTA_SET_NAME];
4759                 set = nft_set_lookup(table, attr, genmask);
4760         }
4761
4762         if (IS_ERR(set)) {
4763                 NL_SET_BAD_ATTR(extack, attr);
4764                 return PTR_ERR(set);
4765         }
4766         if (set->use ||
4767             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
4768              atomic_read(&set->nelems) > 0)) {
4769                 NL_SET_BAD_ATTR(extack, attr);
4770                 return -EBUSY;
4771         }
4772
4773         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4774
4775         return nft_delset(&ctx, set);
4776 }
4777
4778 static int nft_validate_register_store(const struct nft_ctx *ctx,
4779                                        enum nft_registers reg,
4780                                        const struct nft_data *data,
4781                                        enum nft_data_types type,
4782                                        unsigned int len);
4783
4784 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
4785                                      struct nft_set *set,
4786                                      struct nft_set_elem *elem)
4787 {
4788         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
4789         enum nft_registers dreg;
4790
4791         dreg = nft_type_to_reg(set->dtype);
4792         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
4793                                            set->dtype == NFT_DATA_VERDICT ?
4794                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
4795                                            set->dlen);
4796 }
4797
4798 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
4799                                         struct nft_set *set,
4800                                         const struct nft_set_iter *iter,
4801                                         struct nft_set_elem *elem)
4802 {
4803         return nft_setelem_data_validate(ctx, set, elem);
4804 }
4805
4806 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
4807                                        struct nft_set *set)
4808 {
4809         u8 genmask = nft_genmask_next(ctx->net);
4810         struct nft_set_elem_catchall *catchall;
4811         struct nft_set_elem elem;
4812         struct nft_set_ext *ext;
4813         int ret = 0;
4814
4815         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
4816                 ext = nft_set_elem_ext(set, catchall->elem);
4817                 if (!nft_set_elem_active(ext, genmask))
4818                         continue;
4819
4820                 elem.priv = catchall->elem;
4821                 ret = nft_setelem_data_validate(ctx, set, &elem);
4822                 if (ret < 0)
4823                         break;
4824         }
4825
4826         return ret;
4827 }
4828
4829 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
4830                        struct nft_set_binding *binding)
4831 {
4832         struct nft_set_binding *i;
4833         struct nft_set_iter iter;
4834
4835         if (set->use == UINT_MAX)
4836                 return -EOVERFLOW;
4837
4838         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
4839                 return -EBUSY;
4840
4841         if (binding->flags & NFT_SET_MAP) {
4842                 /* If the set is already bound to the same chain all
4843                  * jumps are already validated for that chain.
4844                  */
4845                 list_for_each_entry(i, &set->bindings, list) {
4846                         if (i->flags & NFT_SET_MAP &&
4847                             i->chain == binding->chain)
4848                                 goto bind;
4849                 }
4850
4851                 iter.genmask    = nft_genmask_next(ctx->net);
4852                 iter.skip       = 0;
4853                 iter.count      = 0;
4854                 iter.err        = 0;
4855                 iter.fn         = nf_tables_bind_check_setelem;
4856
4857                 set->ops->walk(ctx, set, &iter);
4858                 if (!iter.err)
4859                         iter.err = nft_set_catchall_bind_check(ctx, set);
4860
4861                 if (iter.err < 0)
4862                         return iter.err;
4863         }
4864 bind:
4865         binding->chain = ctx->chain;
4866         list_add_tail_rcu(&binding->list, &set->bindings);
4867         nft_set_trans_bind(ctx, set);
4868         set->use++;
4869
4870         return 0;
4871 }
4872 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
4873
4874 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
4875                                  struct nft_set_binding *binding, bool event)
4876 {
4877         list_del_rcu(&binding->list);
4878
4879         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
4880                 list_del_rcu(&set->list);
4881                 if (event)
4882                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
4883                                              GFP_KERNEL);
4884         }
4885 }
4886
4887 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
4888                               struct nft_set_binding *binding,
4889                               enum nft_trans_phase phase)
4890 {
4891         switch (phase) {
4892         case NFT_TRANS_PREPARE:
4893                 set->use--;
4894                 return;
4895         case NFT_TRANS_ABORT:
4896         case NFT_TRANS_RELEASE:
4897                 set->use--;
4898                 fallthrough;
4899         default:
4900                 nf_tables_unbind_set(ctx, set, binding,
4901                                      phase == NFT_TRANS_COMMIT);
4902         }
4903 }
4904 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
4905
4906 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
4907 {
4908         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
4909                 nft_set_destroy(ctx, set);
4910 }
4911 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
4912
4913 const struct nft_set_ext_type nft_set_ext_types[] = {
4914         [NFT_SET_EXT_KEY]               = {
4915                 .align  = __alignof__(u32),
4916         },
4917         [NFT_SET_EXT_DATA]              = {
4918                 .align  = __alignof__(u32),
4919         },
4920         [NFT_SET_EXT_EXPRESSIONS]       = {
4921                 .align  = __alignof__(struct nft_set_elem_expr),
4922         },
4923         [NFT_SET_EXT_OBJREF]            = {
4924                 .len    = sizeof(struct nft_object *),
4925                 .align  = __alignof__(struct nft_object *),
4926         },
4927         [NFT_SET_EXT_FLAGS]             = {
4928                 .len    = sizeof(u8),
4929                 .align  = __alignof__(u8),
4930         },
4931         [NFT_SET_EXT_TIMEOUT]           = {
4932                 .len    = sizeof(u64),
4933                 .align  = __alignof__(u64),
4934         },
4935         [NFT_SET_EXT_EXPIRATION]        = {
4936                 .len    = sizeof(u64),
4937                 .align  = __alignof__(u64),
4938         },
4939         [NFT_SET_EXT_USERDATA]          = {
4940                 .len    = sizeof(struct nft_userdata),
4941                 .align  = __alignof__(struct nft_userdata),
4942         },
4943         [NFT_SET_EXT_KEY_END]           = {
4944                 .align  = __alignof__(u32),
4945         },
4946 };
4947
4948 /*
4949  * Set elements
4950  */
4951
4952 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
4953         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
4954         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
4955         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
4956         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
4957         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
4958         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
4959                                             .len = NFT_USERDATA_MAXLEN },
4960         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
4961         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
4962                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
4963         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
4964         [NFTA_SET_ELEM_EXPRESSIONS]     = { .type = NLA_NESTED },
4965 };
4966
4967 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
4968         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
4969                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4970         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
4971                                             .len = NFT_SET_MAXNAMELEN - 1 },
4972         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
4973         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
4974 };
4975
4976 static int nft_set_elem_expr_dump(struct sk_buff *skb,
4977                                   const struct nft_set *set,
4978                                   const struct nft_set_ext *ext)
4979 {
4980         struct nft_set_elem_expr *elem_expr;
4981         u32 size, num_exprs = 0;
4982         struct nft_expr *expr;
4983         struct nlattr *nest;
4984
4985         elem_expr = nft_set_ext_expr(ext);
4986         nft_setelem_expr_foreach(expr, elem_expr, size)
4987                 num_exprs++;
4988
4989         if (num_exprs == 1) {
4990                 expr = nft_setelem_expr_at(elem_expr, 0);
4991                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr) < 0)
4992                         return -1;
4993
4994                 return 0;
4995         } else if (num_exprs > 1) {
4996                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
4997                 if (nest == NULL)
4998                         goto nla_put_failure;
4999
5000                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5001                         expr = nft_setelem_expr_at(elem_expr, size);
5002                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr) < 0)
5003                                 goto nla_put_failure;
5004                 }
5005                 nla_nest_end(skb, nest);
5006         }
5007         return 0;
5008
5009 nla_put_failure:
5010         return -1;
5011 }
5012
5013 static int nf_tables_fill_setelem(struct sk_buff *skb,
5014                                   const struct nft_set *set,
5015                                   const struct nft_set_elem *elem)
5016 {
5017         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5018         unsigned char *b = skb_tail_pointer(skb);
5019         struct nlattr *nest;
5020
5021         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5022         if (nest == NULL)
5023                 goto nla_put_failure;
5024
5025         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5026             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5027                           NFT_DATA_VALUE, set->klen) < 0)
5028                 goto nla_put_failure;
5029
5030         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5031             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5032                           NFT_DATA_VALUE, set->klen) < 0)
5033                 goto nla_put_failure;
5034
5035         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5036             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5037                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5038                           set->dlen) < 0)
5039                 goto nla_put_failure;
5040
5041         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5042             nft_set_elem_expr_dump(skb, set, ext))
5043                 goto nla_put_failure;
5044
5045         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5046             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5047                            (*nft_set_ext_obj(ext))->key.name) < 0)
5048                 goto nla_put_failure;
5049
5050         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5051             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5052                          htonl(*nft_set_ext_flags(ext))))
5053                 goto nla_put_failure;
5054
5055         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
5056             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5057                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
5058                          NFTA_SET_ELEM_PAD))
5059                 goto nla_put_failure;
5060
5061         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5062                 u64 expires, now = get_jiffies_64();
5063
5064                 expires = *nft_set_ext_expiration(ext);
5065                 if (time_before64(now, expires))
5066                         expires -= now;
5067                 else
5068                         expires = 0;
5069
5070                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5071                                  nf_jiffies64_to_msecs(expires),
5072                                  NFTA_SET_ELEM_PAD))
5073                         goto nla_put_failure;
5074         }
5075
5076         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5077                 struct nft_userdata *udata;
5078
5079                 udata = nft_set_ext_userdata(ext);
5080                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5081                             udata->len + 1, udata->data))
5082                         goto nla_put_failure;
5083         }
5084
5085         nla_nest_end(skb, nest);
5086         return 0;
5087
5088 nla_put_failure:
5089         nlmsg_trim(skb, b);
5090         return -EMSGSIZE;
5091 }
5092
5093 struct nft_set_dump_args {
5094         const struct netlink_callback   *cb;
5095         struct nft_set_iter             iter;
5096         struct sk_buff                  *skb;
5097 };
5098
5099 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5100                                   struct nft_set *set,
5101                                   const struct nft_set_iter *iter,
5102                                   struct nft_set_elem *elem)
5103 {
5104         struct nft_set_dump_args *args;
5105
5106         args = container_of(iter, struct nft_set_dump_args, iter);
5107         return nf_tables_fill_setelem(args->skb, set, elem);
5108 }
5109
5110 struct nft_set_dump_ctx {
5111         const struct nft_set    *set;
5112         struct nft_ctx          ctx;
5113 };
5114
5115 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5116                                  const struct nft_set *set)
5117 {
5118         struct nft_set_elem_catchall *catchall;
5119         u8 genmask = nft_genmask_cur(net);
5120         struct nft_set_elem elem;
5121         struct nft_set_ext *ext;
5122         int ret = 0;
5123
5124         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5125                 ext = nft_set_elem_ext(set, catchall->elem);
5126                 if (!nft_set_elem_active(ext, genmask) ||
5127                     nft_set_elem_expired(ext))
5128                         continue;
5129
5130                 elem.priv = catchall->elem;
5131                 ret = nf_tables_fill_setelem(skb, set, &elem);
5132                 break;
5133         }
5134
5135         return ret;
5136 }
5137
5138 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5139 {
5140         struct nft_set_dump_ctx *dump_ctx = cb->data;
5141         struct net *net = sock_net(skb->sk);
5142         struct nftables_pernet *nft_net;
5143         struct nft_table *table;
5144         struct nft_set *set;
5145         struct nft_set_dump_args args;
5146         bool set_found = false;
5147         struct nlmsghdr *nlh;
5148         struct nlattr *nest;
5149         u32 portid, seq;
5150         int event;
5151
5152         rcu_read_lock();
5153         nft_net = nft_pernet(net);
5154         cb->seq = READ_ONCE(nft_net->base_seq);
5155
5156         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5157                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5158                     dump_ctx->ctx.family != table->family)
5159                         continue;
5160
5161                 if (table != dump_ctx->ctx.table)
5162                         continue;
5163
5164                 list_for_each_entry_rcu(set, &table->sets, list) {
5165                         if (set == dump_ctx->set) {
5166                                 set_found = true;
5167                                 break;
5168                         }
5169                 }
5170                 break;
5171         }
5172
5173         if (!set_found) {
5174                 rcu_read_unlock();
5175                 return -ENOENT;
5176         }
5177
5178         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5179         portid = NETLINK_CB(cb->skb).portid;
5180         seq    = cb->nlh->nlmsg_seq;
5181
5182         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5183                            table->family, NFNETLINK_V0, nft_base_seq(net));
5184         if (!nlh)
5185                 goto nla_put_failure;
5186
5187         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5188                 goto nla_put_failure;
5189         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5190                 goto nla_put_failure;
5191
5192         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5193         if (nest == NULL)
5194                 goto nla_put_failure;
5195
5196         args.cb                 = cb;
5197         args.skb                = skb;
5198         args.iter.genmask       = nft_genmask_cur(net);
5199         args.iter.skip          = cb->args[0];
5200         args.iter.count         = 0;
5201         args.iter.err           = 0;
5202         args.iter.fn            = nf_tables_dump_setelem;
5203         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5204
5205         if (!args.iter.err && args.iter.count == cb->args[0])
5206                 args.iter.err = nft_set_catchall_dump(net, skb, set);
5207         rcu_read_unlock();
5208
5209         nla_nest_end(skb, nest);
5210         nlmsg_end(skb, nlh);
5211
5212         if (args.iter.err && args.iter.err != -EMSGSIZE)
5213                 return args.iter.err;
5214         if (args.iter.count == cb->args[0])
5215                 return 0;
5216
5217         cb->args[0] = args.iter.count;
5218         return skb->len;
5219
5220 nla_put_failure:
5221         rcu_read_unlock();
5222         return -ENOSPC;
5223 }
5224
5225 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5226 {
5227         struct nft_set_dump_ctx *dump_ctx = cb->data;
5228
5229         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5230
5231         return cb->data ? 0 : -ENOMEM;
5232 }
5233
5234 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5235 {
5236         kfree(cb->data);
5237         return 0;
5238 }
5239
5240 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5241                                        const struct nft_ctx *ctx, u32 seq,
5242                                        u32 portid, int event, u16 flags,
5243                                        const struct nft_set *set,
5244                                        const struct nft_set_elem *elem)
5245 {
5246         struct nlmsghdr *nlh;
5247         struct nlattr *nest;
5248         int err;
5249
5250         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5251         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
5252                            NFNETLINK_V0, nft_base_seq(ctx->net));
5253         if (!nlh)
5254                 goto nla_put_failure;
5255
5256         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
5257                 goto nla_put_failure;
5258         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
5259                 goto nla_put_failure;
5260
5261         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5262         if (nest == NULL)
5263                 goto nla_put_failure;
5264
5265         err = nf_tables_fill_setelem(skb, set, elem);
5266         if (err < 0)
5267                 goto nla_put_failure;
5268
5269         nla_nest_end(skb, nest);
5270
5271         nlmsg_end(skb, nlh);
5272         return 0;
5273
5274 nla_put_failure:
5275         nlmsg_trim(skb, nlh);
5276         return -1;
5277 }
5278
5279 static int nft_setelem_parse_flags(const struct nft_set *set,
5280                                    const struct nlattr *attr, u32 *flags)
5281 {
5282         if (attr == NULL)
5283                 return 0;
5284
5285         *flags = ntohl(nla_get_be32(attr));
5286         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5287                 return -EOPNOTSUPP;
5288         if (!(set->flags & NFT_SET_INTERVAL) &&
5289             *flags & NFT_SET_ELEM_INTERVAL_END)
5290                 return -EINVAL;
5291         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
5292             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5293                 return -EINVAL;
5294
5295         return 0;
5296 }
5297
5298 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
5299                                  struct nft_data *key, struct nlattr *attr)
5300 {
5301         struct nft_data_desc desc = {
5302                 .type   = NFT_DATA_VALUE,
5303                 .size   = NFT_DATA_VALUE_MAXLEN,
5304                 .len    = set->klen,
5305         };
5306
5307         return nft_data_init(ctx, key, &desc, attr);
5308 }
5309
5310 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5311                                   struct nft_data_desc *desc,
5312                                   struct nft_data *data,
5313                                   struct nlattr *attr)
5314 {
5315         u32 dtype;
5316
5317         if (set->dtype == NFT_DATA_VERDICT)
5318                 dtype = NFT_DATA_VERDICT;
5319         else
5320                 dtype = NFT_DATA_VALUE;
5321
5322         desc->type = dtype;
5323         desc->size = NFT_DATA_VALUE_MAXLEN;
5324         desc->len = set->dlen;
5325         desc->flags = NFT_DATA_DESC_SETELEM;
5326
5327         return nft_data_init(ctx, data, desc, attr);
5328 }
5329
5330 static void *nft_setelem_catchall_get(const struct net *net,
5331                                       const struct nft_set *set)
5332 {
5333         struct nft_set_elem_catchall *catchall;
5334         u8 genmask = nft_genmask_cur(net);
5335         struct nft_set_ext *ext;
5336         void *priv = NULL;
5337
5338         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5339                 ext = nft_set_elem_ext(set, catchall->elem);
5340                 if (!nft_set_elem_active(ext, genmask) ||
5341                     nft_set_elem_expired(ext))
5342                         continue;
5343
5344                 priv = catchall->elem;
5345                 break;
5346         }
5347
5348         return priv;
5349 }
5350
5351 static int nft_setelem_get(struct nft_ctx *ctx, struct nft_set *set,
5352                            struct nft_set_elem *elem, u32 flags)
5353 {
5354         void *priv;
5355
5356         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
5357                 priv = set->ops->get(ctx->net, set, elem, flags);
5358                 if (IS_ERR(priv))
5359                         return PTR_ERR(priv);
5360         } else {
5361                 priv = nft_setelem_catchall_get(ctx->net, set);
5362                 if (!priv)
5363                         return -ENOENT;
5364         }
5365         elem->priv = priv;
5366
5367         return 0;
5368 }
5369
5370 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5371                             const struct nlattr *attr)
5372 {
5373         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5374         struct nft_set_elem elem;
5375         struct sk_buff *skb;
5376         uint32_t flags = 0;
5377         int err;
5378
5379         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5380                                           nft_set_elem_policy, NULL);
5381         if (err < 0)
5382                 return err;
5383
5384         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5385         if (err < 0)
5386                 return err;
5387
5388         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5389                 return -EINVAL;
5390
5391         if (nla[NFTA_SET_ELEM_KEY]) {
5392                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5393                                             nla[NFTA_SET_ELEM_KEY]);
5394                 if (err < 0)
5395                         return err;
5396         }
5397
5398         if (nla[NFTA_SET_ELEM_KEY_END]) {
5399                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5400                                             nla[NFTA_SET_ELEM_KEY_END]);
5401                 if (err < 0)
5402                         return err;
5403         }
5404
5405         err = nft_setelem_get(ctx, set, &elem, flags);
5406         if (err < 0)
5407                 return err;
5408
5409         err = -ENOMEM;
5410         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
5411         if (skb == NULL)
5412                 return err;
5413
5414         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
5415                                           NFT_MSG_NEWSETELEM, 0, set, &elem);
5416         if (err < 0)
5417                 goto err_fill_setelem;
5418
5419         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
5420
5421 err_fill_setelem:
5422         kfree_skb(skb);
5423         return err;
5424 }
5425
5426 /* called with rcu_read_lock held */
5427 static int nf_tables_getsetelem(struct sk_buff *skb,
5428                                 const struct nfnl_info *info,
5429                                 const struct nlattr * const nla[])
5430 {
5431         struct netlink_ext_ack *extack = info->extack;
5432         u8 genmask = nft_genmask_cur(info->net);
5433         u8 family = info->nfmsg->nfgen_family;
5434         struct net *net = info->net;
5435         struct nft_table *table;
5436         struct nft_set *set;
5437         struct nlattr *attr;
5438         struct nft_ctx ctx;
5439         int rem, err = 0;
5440
5441         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
5442                                  genmask, 0);
5443         if (IS_ERR(table)) {
5444                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
5445                 return PTR_ERR(table);
5446         }
5447
5448         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
5449         if (IS_ERR(set))
5450                 return PTR_ERR(set);
5451
5452         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5453
5454         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
5455                 struct netlink_dump_control c = {
5456                         .start = nf_tables_dump_set_start,
5457                         .dump = nf_tables_dump_set,
5458                         .done = nf_tables_dump_set_done,
5459                         .module = THIS_MODULE,
5460                 };
5461                 struct nft_set_dump_ctx dump_ctx = {
5462                         .set = set,
5463                         .ctx = ctx,
5464                 };
5465
5466                 c.data = &dump_ctx;
5467                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
5468         }
5469
5470         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
5471                 return -EINVAL;
5472
5473         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
5474                 err = nft_get_set_elem(&ctx, set, attr);
5475                 if (err < 0) {
5476                         NL_SET_BAD_ATTR(extack, attr);
5477                         break;
5478                 }
5479         }
5480
5481         return err;
5482 }
5483
5484 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
5485                                      const struct nft_set *set,
5486                                      const struct nft_set_elem *elem,
5487                                      int event)
5488 {
5489         struct nftables_pernet *nft_net;
5490         struct net *net = ctx->net;
5491         u32 portid = ctx->portid;
5492         struct sk_buff *skb;
5493         u16 flags = 0;
5494         int err;
5495
5496         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
5497                 return;
5498
5499         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
5500         if (skb == NULL)
5501                 goto err;
5502
5503         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
5504                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
5505
5506         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
5507                                           set, elem);
5508         if (err < 0) {
5509                 kfree_skb(skb);
5510                 goto err;
5511         }
5512
5513         nft_net = nft_pernet(net);
5514         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
5515         return;
5516 err:
5517         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
5518 }
5519
5520 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
5521                                               int msg_type,
5522                                               struct nft_set *set)
5523 {
5524         struct nft_trans *trans;
5525
5526         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
5527         if (trans == NULL)
5528                 return NULL;
5529
5530         nft_trans_elem_set(trans) = set;
5531         return trans;
5532 }
5533
5534 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
5535                                          const struct nft_set *set,
5536                                          const struct nlattr *attr)
5537 {
5538         struct nft_expr *expr;
5539         int err;
5540
5541         expr = nft_expr_init(ctx, attr);
5542         if (IS_ERR(expr))
5543                 return expr;
5544
5545         err = -EOPNOTSUPP;
5546         if (expr->ops->type->flags & NFT_EXPR_GC) {
5547                 if (set->flags & NFT_SET_TIMEOUT)
5548                         goto err_set_elem_expr;
5549                 if (!set->ops->gc_init)
5550                         goto err_set_elem_expr;
5551                 set->ops->gc_init(set);
5552         }
5553
5554         return expr;
5555
5556 err_set_elem_expr:
5557         nft_expr_destroy(ctx, expr);
5558         return ERR_PTR(err);
5559 }
5560
5561 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
5562 {
5563         len += nft_set_ext_types[id].len;
5564         if (len > tmpl->ext_len[id] ||
5565             len > U8_MAX)
5566                 return -1;
5567
5568         return 0;
5569 }
5570
5571 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
5572                               void *to, const void *from, u32 len)
5573 {
5574         if (nft_set_ext_check(tmpl, id, len) < 0)
5575                 return -1;
5576
5577         memcpy(to, from, len);
5578
5579         return 0;
5580 }
5581
5582 void *nft_set_elem_init(const struct nft_set *set,
5583                         const struct nft_set_ext_tmpl *tmpl,
5584                         const u32 *key, const u32 *key_end,
5585                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
5586 {
5587         struct nft_set_ext *ext;
5588         void *elem;
5589
5590         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
5591         if (elem == NULL)
5592                 return ERR_PTR(-ENOMEM);
5593
5594         ext = nft_set_elem_ext(set, elem);
5595         nft_set_ext_init(ext, tmpl);
5596
5597         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5598             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
5599                                nft_set_ext_key(ext), key, set->klen) < 0)
5600                 goto err_ext_check;
5601
5602         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5603             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
5604                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
5605                 goto err_ext_check;
5606
5607         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5608             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
5609                                nft_set_ext_data(ext), data, set->dlen) < 0)
5610                 goto err_ext_check;
5611
5612         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5613                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
5614                 if (expiration == 0)
5615                         *nft_set_ext_expiration(ext) += timeout;
5616         }
5617         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
5618                 *nft_set_ext_timeout(ext) = timeout;
5619
5620         return elem;
5621
5622 err_ext_check:
5623         kfree(elem);
5624
5625         return ERR_PTR(-EINVAL);
5626 }
5627
5628 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5629                                         struct nft_expr *expr)
5630 {
5631         if (expr->ops->destroy_clone) {
5632                 expr->ops->destroy_clone(ctx, expr);
5633                 module_put(expr->ops->type->owner);
5634         } else {
5635                 nf_tables_expr_destroy(ctx, expr);
5636         }
5637 }
5638
5639 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5640                                       struct nft_set_elem_expr *elem_expr)
5641 {
5642         struct nft_expr *expr;
5643         u32 size;
5644
5645         nft_setelem_expr_foreach(expr, elem_expr, size)
5646                 __nft_set_elem_expr_destroy(ctx, expr);
5647 }
5648
5649 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
5650                           bool destroy_expr)
5651 {
5652         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5653         struct nft_ctx ctx = {
5654                 .net    = read_pnet(&set->net),
5655                 .family = set->table->family,
5656         };
5657
5658         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
5659         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
5660                 nft_data_release(nft_set_ext_data(ext), set->dtype);
5661         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
5662                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
5663
5664         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
5665                 (*nft_set_ext_obj(ext))->use--;
5666         kfree(elem);
5667 }
5668 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
5669
5670 /* Only called from commit path, nft_setelem_data_deactivate() already deals
5671  * with the refcounting from the preparation phase.
5672  */
5673 static void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
5674                                        const struct nft_set *set, void *elem)
5675 {
5676         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5677
5678         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
5679                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
5680
5681         kfree(elem);
5682 }
5683
5684 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
5685                             struct nft_expr *expr_array[])
5686 {
5687         struct nft_expr *expr;
5688         int err, i, k;
5689
5690         for (i = 0; i < set->num_exprs; i++) {
5691                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
5692                 if (!expr)
5693                         goto err_expr;
5694
5695                 err = nft_expr_clone(expr, set->exprs[i]);
5696                 if (err < 0) {
5697                         kfree(expr);
5698                         goto err_expr;
5699                 }
5700                 expr_array[i] = expr;
5701         }
5702
5703         return 0;
5704
5705 err_expr:
5706         for (k = i - 1; k >= 0; k--)
5707                 nft_expr_destroy(ctx, expr_array[k]);
5708
5709         return -ENOMEM;
5710 }
5711
5712 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
5713                                    const struct nft_set_ext_tmpl *tmpl,
5714                                    const struct nft_set_ext *ext,
5715                                    struct nft_expr *expr_array[],
5716                                    u32 num_exprs)
5717 {
5718         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
5719         u32 len = sizeof(struct nft_set_elem_expr);
5720         struct nft_expr *expr;
5721         int i, err;
5722
5723         if (num_exprs == 0)
5724                 return 0;
5725
5726         for (i = 0; i < num_exprs; i++)
5727                 len += expr_array[i]->ops->size;
5728
5729         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
5730                 return -EINVAL;
5731
5732         for (i = 0; i < num_exprs; i++) {
5733                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
5734                 err = nft_expr_clone(expr, expr_array[i]);
5735                 if (err < 0)
5736                         goto err_elem_expr_setup;
5737
5738                 elem_expr->size += expr_array[i]->ops->size;
5739                 nft_expr_destroy(ctx, expr_array[i]);
5740                 expr_array[i] = NULL;
5741         }
5742
5743         return 0;
5744
5745 err_elem_expr_setup:
5746         for (; i < num_exprs; i++) {
5747                 nft_expr_destroy(ctx, expr_array[i]);
5748                 expr_array[i] = NULL;
5749         }
5750
5751         return -ENOMEM;
5752 }
5753
5754 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
5755                                             const struct nft_set *set)
5756 {
5757         struct nft_set_elem_catchall *catchall;
5758         u8 genmask = nft_genmask_cur(net);
5759         struct nft_set_ext *ext;
5760
5761         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5762                 ext = nft_set_elem_ext(set, catchall->elem);
5763                 if (nft_set_elem_active(ext, genmask) &&
5764                     !nft_set_elem_expired(ext))
5765                         return ext;
5766         }
5767
5768         return NULL;
5769 }
5770 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
5771
5772 void *nft_set_catchall_gc(const struct nft_set *set)
5773 {
5774         struct nft_set_elem_catchall *catchall, *next;
5775         struct nft_set_ext *ext;
5776         void *elem = NULL;
5777
5778         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5779                 ext = nft_set_elem_ext(set, catchall->elem);
5780
5781                 if (!nft_set_elem_expired(ext) ||
5782                     nft_set_elem_mark_busy(ext))
5783                         continue;
5784
5785                 elem = catchall->elem;
5786                 list_del_rcu(&catchall->list);
5787                 kfree_rcu(catchall, rcu);
5788                 break;
5789         }
5790
5791         return elem;
5792 }
5793 EXPORT_SYMBOL_GPL(nft_set_catchall_gc);
5794
5795 static int nft_setelem_catchall_insert(const struct net *net,
5796                                        struct nft_set *set,
5797                                        const struct nft_set_elem *elem,
5798                                        struct nft_set_ext **pext)
5799 {
5800         struct nft_set_elem_catchall *catchall;
5801         u8 genmask = nft_genmask_next(net);
5802         struct nft_set_ext *ext;
5803
5804         list_for_each_entry(catchall, &set->catchall_list, list) {
5805                 ext = nft_set_elem_ext(set, catchall->elem);
5806                 if (nft_set_elem_active(ext, genmask)) {
5807                         *pext = ext;
5808                         return -EEXIST;
5809                 }
5810         }
5811
5812         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
5813         if (!catchall)
5814                 return -ENOMEM;
5815
5816         catchall->elem = elem->priv;
5817         list_add_tail_rcu(&catchall->list, &set->catchall_list);
5818
5819         return 0;
5820 }
5821
5822 static int nft_setelem_insert(const struct net *net,
5823                               struct nft_set *set,
5824                               const struct nft_set_elem *elem,
5825                               struct nft_set_ext **ext, unsigned int flags)
5826 {
5827         int ret;
5828
5829         if (flags & NFT_SET_ELEM_CATCHALL)
5830                 ret = nft_setelem_catchall_insert(net, set, elem, ext);
5831         else
5832                 ret = set->ops->insert(net, set, elem, ext);
5833
5834         return ret;
5835 }
5836
5837 static bool nft_setelem_is_catchall(const struct nft_set *set,
5838                                     const struct nft_set_elem *elem)
5839 {
5840         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5841
5842         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5843             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
5844                 return true;
5845
5846         return false;
5847 }
5848
5849 static void nft_setelem_activate(struct net *net, struct nft_set *set,
5850                                  struct nft_set_elem *elem)
5851 {
5852         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5853
5854         if (nft_setelem_is_catchall(set, elem)) {
5855                 nft_set_elem_change_active(net, set, ext);
5856                 nft_set_elem_clear_busy(ext);
5857         } else {
5858                 set->ops->activate(net, set, elem);
5859         }
5860 }
5861
5862 static int nft_setelem_catchall_deactivate(const struct net *net,
5863                                            struct nft_set *set,
5864                                            struct nft_set_elem *elem)
5865 {
5866         struct nft_set_elem_catchall *catchall;
5867         struct nft_set_ext *ext;
5868
5869         list_for_each_entry(catchall, &set->catchall_list, list) {
5870                 ext = nft_set_elem_ext(set, catchall->elem);
5871                 if (!nft_is_active(net, ext) ||
5872                     nft_set_elem_mark_busy(ext))
5873                         continue;
5874
5875                 kfree(elem->priv);
5876                 elem->priv = catchall->elem;
5877                 nft_set_elem_change_active(net, set, ext);
5878                 return 0;
5879         }
5880
5881         return -ENOENT;
5882 }
5883
5884 static int __nft_setelem_deactivate(const struct net *net,
5885                                     struct nft_set *set,
5886                                     struct nft_set_elem *elem)
5887 {
5888         void *priv;
5889
5890         priv = set->ops->deactivate(net, set, elem);
5891         if (!priv)
5892                 return -ENOENT;
5893
5894         kfree(elem->priv);
5895         elem->priv = priv;
5896         set->ndeact++;
5897
5898         return 0;
5899 }
5900
5901 static int nft_setelem_deactivate(const struct net *net,
5902                                   struct nft_set *set,
5903                                   struct nft_set_elem *elem, u32 flags)
5904 {
5905         int ret;
5906
5907         if (flags & NFT_SET_ELEM_CATCHALL)
5908                 ret = nft_setelem_catchall_deactivate(net, set, elem);
5909         else
5910                 ret = __nft_setelem_deactivate(net, set, elem);
5911
5912         return ret;
5913 }
5914
5915 static void nft_setelem_catchall_remove(const struct net *net,
5916                                         const struct nft_set *set,
5917                                         const struct nft_set_elem *elem)
5918 {
5919         struct nft_set_elem_catchall *catchall, *next;
5920
5921         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5922                 if (catchall->elem == elem->priv) {
5923                         list_del_rcu(&catchall->list);
5924                         kfree_rcu(catchall, rcu);
5925                         break;
5926                 }
5927         }
5928 }
5929
5930 static void nft_setelem_remove(const struct net *net,
5931                                const struct nft_set *set,
5932                                const struct nft_set_elem *elem)
5933 {
5934         if (nft_setelem_is_catchall(set, elem))
5935                 nft_setelem_catchall_remove(net, set, elem);
5936         else
5937                 set->ops->remove(net, set, elem);
5938 }
5939
5940 static bool nft_setelem_valid_key_end(const struct nft_set *set,
5941                                       struct nlattr **nla, u32 flags)
5942 {
5943         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
5944                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
5945                 if (flags & NFT_SET_ELEM_INTERVAL_END)
5946                         return false;
5947
5948                 if (nla[NFTA_SET_ELEM_KEY_END] &&
5949                     flags & NFT_SET_ELEM_CATCHALL)
5950                         return false;
5951         } else {
5952                 if (nla[NFTA_SET_ELEM_KEY_END])
5953                         return false;
5954         }
5955
5956         return true;
5957 }
5958
5959 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5960                             const struct nlattr *attr, u32 nlmsg_flags)
5961 {
5962         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
5963         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5964         u8 genmask = nft_genmask_next(ctx->net);
5965         u32 flags = 0, size = 0, num_exprs = 0;
5966         struct nft_set_ext_tmpl tmpl;
5967         struct nft_set_ext *ext, *ext2;
5968         struct nft_set_elem elem;
5969         struct nft_set_binding *binding;
5970         struct nft_object *obj = NULL;
5971         struct nft_userdata *udata;
5972         struct nft_data_desc desc;
5973         enum nft_registers dreg;
5974         struct nft_trans *trans;
5975         u64 timeout;
5976         u64 expiration;
5977         int err, i;
5978         u8 ulen;
5979
5980         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5981                                           nft_set_elem_policy, NULL);
5982         if (err < 0)
5983                 return err;
5984
5985         nft_set_ext_prepare(&tmpl);
5986
5987         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5988         if (err < 0)
5989                 return err;
5990
5991         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5992                 return -EINVAL;
5993
5994         if (flags != 0) {
5995                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
5996                 if (err < 0)
5997                         return err;
5998         }
5999
6000         if (set->flags & NFT_SET_MAP) {
6001                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6002                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6003                         return -EINVAL;
6004         } else {
6005                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6006                         return -EINVAL;
6007         }
6008
6009         if (set->flags & NFT_SET_OBJECT) {
6010                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6011                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6012                         return -EINVAL;
6013         } else {
6014                 if (nla[NFTA_SET_ELEM_OBJREF])
6015                         return -EINVAL;
6016         }
6017
6018         if (!nft_setelem_valid_key_end(set, nla, flags))
6019                 return -EINVAL;
6020
6021         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6022              (nla[NFTA_SET_ELEM_DATA] ||
6023               nla[NFTA_SET_ELEM_OBJREF] ||
6024               nla[NFTA_SET_ELEM_TIMEOUT] ||
6025               nla[NFTA_SET_ELEM_EXPIRATION] ||
6026               nla[NFTA_SET_ELEM_USERDATA] ||
6027               nla[NFTA_SET_ELEM_EXPR] ||
6028               nla[NFTA_SET_ELEM_KEY_END] ||
6029               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6030                 return -EINVAL;
6031
6032         timeout = 0;
6033         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6034                 if (!(set->flags & NFT_SET_TIMEOUT))
6035                         return -EINVAL;
6036                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6037                                             &timeout);
6038                 if (err)
6039                         return err;
6040         } else if (set->flags & NFT_SET_TIMEOUT &&
6041                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6042                 timeout = READ_ONCE(set->timeout);
6043         }
6044
6045         expiration = 0;
6046         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6047                 if (!(set->flags & NFT_SET_TIMEOUT))
6048                         return -EINVAL;
6049                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6050                                             &expiration);
6051                 if (err)
6052                         return err;
6053         }
6054
6055         if (nla[NFTA_SET_ELEM_EXPR]) {
6056                 struct nft_expr *expr;
6057
6058                 if (set->num_exprs && set->num_exprs != 1)
6059                         return -EOPNOTSUPP;
6060
6061                 expr = nft_set_elem_expr_alloc(ctx, set,
6062                                                nla[NFTA_SET_ELEM_EXPR]);
6063                 if (IS_ERR(expr))
6064                         return PTR_ERR(expr);
6065
6066                 expr_array[0] = expr;
6067                 num_exprs = 1;
6068
6069                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6070                         err = -EOPNOTSUPP;
6071                         goto err_set_elem_expr;
6072                 }
6073         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6074                 struct nft_expr *expr;
6075                 struct nlattr *tmp;
6076                 int left;
6077
6078                 i = 0;
6079                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6080                         if (i == NFT_SET_EXPR_MAX ||
6081                             (set->num_exprs && set->num_exprs == i)) {
6082                                 err = -E2BIG;
6083                                 goto err_set_elem_expr;
6084                         }
6085                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6086                                 err = -EINVAL;
6087                                 goto err_set_elem_expr;
6088                         }
6089                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6090                         if (IS_ERR(expr)) {
6091                                 err = PTR_ERR(expr);
6092                                 goto err_set_elem_expr;
6093                         }
6094                         expr_array[i] = expr;
6095                         num_exprs++;
6096
6097                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6098                                 err = -EOPNOTSUPP;
6099                                 goto err_set_elem_expr;
6100                         }
6101                         i++;
6102                 }
6103                 if (set->num_exprs && set->num_exprs != i) {
6104                         err = -EOPNOTSUPP;
6105                         goto err_set_elem_expr;
6106                 }
6107         } else if (set->num_exprs > 0 &&
6108                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6109                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6110                 if (err < 0)
6111                         goto err_set_elem_expr_clone;
6112
6113                 num_exprs = set->num_exprs;
6114         }
6115
6116         if (nla[NFTA_SET_ELEM_KEY]) {
6117                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6118                                             nla[NFTA_SET_ELEM_KEY]);
6119                 if (err < 0)
6120                         goto err_set_elem_expr;
6121
6122                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6123                 if (err < 0)
6124                         goto err_parse_key;
6125         }
6126
6127         if (nla[NFTA_SET_ELEM_KEY_END]) {
6128                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6129                                             nla[NFTA_SET_ELEM_KEY_END]);
6130                 if (err < 0)
6131                         goto err_parse_key;
6132
6133                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6134                 if (err < 0)
6135                         goto err_parse_key_end;
6136         }
6137
6138         if (timeout > 0) {
6139                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6140                 if (err < 0)
6141                         goto err_parse_key_end;
6142
6143                 if (timeout != READ_ONCE(set->timeout)) {
6144                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6145                         if (err < 0)
6146                                 goto err_parse_key_end;
6147                 }
6148         }
6149
6150         if (num_exprs) {
6151                 for (i = 0; i < num_exprs; i++)
6152                         size += expr_array[i]->ops->size;
6153
6154                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6155                                              sizeof(struct nft_set_elem_expr) + size);
6156                 if (err < 0)
6157                         goto err_parse_key_end;
6158         }
6159
6160         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6161                 obj = nft_obj_lookup(ctx->net, ctx->table,
6162                                      nla[NFTA_SET_ELEM_OBJREF],
6163                                      set->objtype, genmask);
6164                 if (IS_ERR(obj)) {
6165                         err = PTR_ERR(obj);
6166                         goto err_parse_key_end;
6167                 }
6168                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6169                 if (err < 0)
6170                         goto err_parse_key_end;
6171         }
6172
6173         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6174                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
6175                                              nla[NFTA_SET_ELEM_DATA]);
6176                 if (err < 0)
6177                         goto err_parse_key_end;
6178
6179                 dreg = nft_type_to_reg(set->dtype);
6180                 list_for_each_entry(binding, &set->bindings, list) {
6181                         struct nft_ctx bind_ctx = {
6182                                 .net    = ctx->net,
6183                                 .family = ctx->family,
6184                                 .table  = ctx->table,
6185                                 .chain  = (struct nft_chain *)binding->chain,
6186                         };
6187
6188                         if (!(binding->flags & NFT_SET_MAP))
6189                                 continue;
6190
6191                         err = nft_validate_register_store(&bind_ctx, dreg,
6192                                                           &elem.data.val,
6193                                                           desc.type, desc.len);
6194                         if (err < 0)
6195                                 goto err_parse_data;
6196
6197                         if (desc.type == NFT_DATA_VERDICT &&
6198                             (elem.data.val.verdict.code == NFT_GOTO ||
6199                              elem.data.val.verdict.code == NFT_JUMP))
6200                                 nft_validate_state_update(ctx->net,
6201                                                           NFT_VALIDATE_NEED);
6202                 }
6203
6204                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
6205                 if (err < 0)
6206                         goto err_parse_data;
6207         }
6208
6209         /* The full maximum length of userdata can exceed the maximum
6210          * offset value (U8_MAX) for following extensions, therefor it
6211          * must be the last extension added.
6212          */
6213         ulen = 0;
6214         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
6215                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
6216                 if (ulen > 0) {
6217                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
6218                                                      ulen);
6219                         if (err < 0)
6220                                 goto err_parse_data;
6221                 }
6222         }
6223
6224         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6225                                       elem.key_end.val.data, elem.data.val.data,
6226                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
6227         if (IS_ERR(elem.priv)) {
6228                 err = PTR_ERR(elem.priv);
6229                 goto err_parse_data;
6230         }
6231
6232         ext = nft_set_elem_ext(set, elem.priv);
6233         if (flags)
6234                 *nft_set_ext_flags(ext) = flags;
6235
6236         if (ulen > 0) {
6237                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
6238                         err = -EINVAL;
6239                         goto err_elem_userdata;
6240                 }
6241                 udata = nft_set_ext_userdata(ext);
6242                 udata->len = ulen - 1;
6243                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
6244         }
6245         if (obj) {
6246                 *nft_set_ext_obj(ext) = obj;
6247                 obj->use++;
6248         }
6249         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
6250         if (err < 0)
6251                 goto err_elem_free;
6252
6253         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
6254         if (trans == NULL) {
6255                 err = -ENOMEM;
6256                 goto err_elem_free;
6257         }
6258
6259         ext->genmask = nft_genmask_cur(ctx->net) | NFT_SET_ELEM_BUSY_MASK;
6260
6261         err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
6262         if (err) {
6263                 if (err == -EEXIST) {
6264                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
6265                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
6266                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
6267                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
6268                                 goto err_element_clash;
6269                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6270                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
6271                              memcmp(nft_set_ext_data(ext),
6272                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
6273                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
6274                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
6275                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
6276                                 goto err_element_clash;
6277                         else if (!(nlmsg_flags & NLM_F_EXCL))
6278                                 err = 0;
6279                 } else if (err == -ENOTEMPTY) {
6280                         /* ENOTEMPTY reports overlapping between this element
6281                          * and an existing one.
6282                          */
6283                         err = -EEXIST;
6284                 }
6285                 goto err_element_clash;
6286         }
6287
6288         if (!(flags & NFT_SET_ELEM_CATCHALL) && set->size &&
6289             !atomic_add_unless(&set->nelems, 1, set->size + set->ndeact)) {
6290                 err = -ENFILE;
6291                 goto err_set_full;
6292         }
6293
6294         nft_trans_elem(trans) = elem;
6295         nft_trans_commit_list_add_tail(ctx->net, trans);
6296         return 0;
6297
6298 err_set_full:
6299         nft_setelem_remove(ctx->net, set, &elem);
6300 err_element_clash:
6301         kfree(trans);
6302 err_elem_free:
6303         if (obj)
6304                 obj->use--;
6305 err_elem_userdata:
6306         nf_tables_set_elem_destroy(ctx, set, elem.priv);
6307 err_parse_data:
6308         if (nla[NFTA_SET_ELEM_DATA] != NULL)
6309                 nft_data_release(&elem.data.val, desc.type);
6310 err_parse_key_end:
6311         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6312 err_parse_key:
6313         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6314 err_set_elem_expr:
6315         for (i = 0; i < num_exprs && expr_array[i]; i++)
6316                 nft_expr_destroy(ctx, expr_array[i]);
6317 err_set_elem_expr_clone:
6318         return err;
6319 }
6320
6321 static int nf_tables_newsetelem(struct sk_buff *skb,
6322                                 const struct nfnl_info *info,
6323                                 const struct nlattr * const nla[])
6324 {
6325         struct nftables_pernet *nft_net = nft_pernet(info->net);
6326         struct netlink_ext_ack *extack = info->extack;
6327         u8 genmask = nft_genmask_next(info->net);
6328         u8 family = info->nfmsg->nfgen_family;
6329         struct net *net = info->net;
6330         const struct nlattr *attr;
6331         struct nft_table *table;
6332         struct nft_set *set;
6333         struct nft_ctx ctx;
6334         int rem, err;
6335
6336         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
6337                 return -EINVAL;
6338
6339         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6340                                  genmask, NETLINK_CB(skb).portid);
6341         if (IS_ERR(table)) {
6342                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6343                 return PTR_ERR(table);
6344         }
6345
6346         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
6347                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
6348         if (IS_ERR(set))
6349                 return PTR_ERR(set);
6350
6351         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
6352                 return -EBUSY;
6353
6354         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6355
6356         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6357                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
6358                 if (err < 0) {
6359                         NL_SET_BAD_ATTR(extack, attr);
6360                         return err;
6361                 }
6362         }
6363
6364         if (nft_net->validate_state == NFT_VALIDATE_DO)
6365                 return nft_table_validate(net, table);
6366
6367         return 0;
6368 }
6369
6370 /**
6371  *      nft_data_hold - hold a nft_data item
6372  *
6373  *      @data: struct nft_data to release
6374  *      @type: type of data
6375  *
6376  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
6377  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
6378  *      NFT_GOTO verdicts. This function must be called on active data objects
6379  *      from the second phase of the commit protocol.
6380  */
6381 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
6382 {
6383         struct nft_chain *chain;
6384         struct nft_rule *rule;
6385
6386         if (type == NFT_DATA_VERDICT) {
6387                 switch (data->verdict.code) {
6388                 case NFT_JUMP:
6389                 case NFT_GOTO:
6390                         chain = data->verdict.chain;
6391                         chain->use++;
6392
6393                         if (!nft_chain_is_bound(chain))
6394                                 break;
6395
6396                         chain->table->use++;
6397                         list_for_each_entry(rule, &chain->rules, list)
6398                                 chain->use++;
6399
6400                         nft_chain_add(chain->table, chain);
6401                         break;
6402                 }
6403         }
6404 }
6405
6406 static void nft_setelem_data_activate(const struct net *net,
6407                                       const struct nft_set *set,
6408                                       struct nft_set_elem *elem)
6409 {
6410         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6411
6412         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6413                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
6414         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6415                 (*nft_set_ext_obj(ext))->use++;
6416 }
6417
6418 static void nft_setelem_data_deactivate(const struct net *net,
6419                                         const struct nft_set *set,
6420                                         struct nft_set_elem *elem)
6421 {
6422         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6423
6424         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6425                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6426         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6427                 (*nft_set_ext_obj(ext))->use--;
6428 }
6429
6430 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
6431                            const struct nlattr *attr)
6432 {
6433         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6434         struct nft_set_ext_tmpl tmpl;
6435         struct nft_set_elem elem;
6436         struct nft_set_ext *ext;
6437         struct nft_trans *trans;
6438         u32 flags = 0;
6439         int err;
6440
6441         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6442                                           nft_set_elem_policy, NULL);
6443         if (err < 0)
6444                 return err;
6445
6446         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6447         if (err < 0)
6448                 return err;
6449
6450         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6451                 return -EINVAL;
6452
6453         if (!nft_setelem_valid_key_end(set, nla, flags))
6454                 return -EINVAL;
6455
6456         nft_set_ext_prepare(&tmpl);
6457
6458         if (flags != 0) {
6459                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6460                 if (err < 0)
6461                         return err;
6462         }
6463
6464         if (nla[NFTA_SET_ELEM_KEY]) {
6465                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6466                                             nla[NFTA_SET_ELEM_KEY]);
6467                 if (err < 0)
6468                         return err;
6469
6470                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6471                 if (err < 0)
6472                         goto fail_elem;
6473         }
6474
6475         if (nla[NFTA_SET_ELEM_KEY_END]) {
6476                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6477                                             nla[NFTA_SET_ELEM_KEY_END]);
6478                 if (err < 0)
6479                         goto fail_elem;
6480
6481                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6482                 if (err < 0)
6483                         goto fail_elem_key_end;
6484         }
6485
6486         err = -ENOMEM;
6487         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6488                                       elem.key_end.val.data, NULL, 0, 0,
6489                                       GFP_KERNEL_ACCOUNT);
6490         if (IS_ERR(elem.priv)) {
6491                 err = PTR_ERR(elem.priv);
6492                 goto fail_elem_key_end;
6493         }
6494
6495         ext = nft_set_elem_ext(set, elem.priv);
6496         if (flags)
6497                 *nft_set_ext_flags(ext) = flags;
6498
6499         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
6500         if (trans == NULL)
6501                 goto fail_trans;
6502
6503         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
6504         if (err < 0)
6505                 goto fail_ops;
6506
6507         nft_setelem_data_deactivate(ctx->net, set, &elem);
6508
6509         nft_trans_elem(trans) = elem;
6510         nft_trans_commit_list_add_tail(ctx->net, trans);
6511         return 0;
6512
6513 fail_ops:
6514         kfree(trans);
6515 fail_trans:
6516         kfree(elem.priv);
6517 fail_elem_key_end:
6518         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
6519 fail_elem:
6520         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6521         return err;
6522 }
6523
6524 static int nft_setelem_flush(const struct nft_ctx *ctx,
6525                              struct nft_set *set,
6526                              const struct nft_set_iter *iter,
6527                              struct nft_set_elem *elem)
6528 {
6529         struct nft_trans *trans;
6530         int err;
6531
6532         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6533                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
6534         if (!trans)
6535                 return -ENOMEM;
6536
6537         if (!set->ops->flush(ctx->net, set, elem->priv)) {
6538                 err = -ENOENT;
6539                 goto err1;
6540         }
6541         set->ndeact++;
6542
6543         nft_setelem_data_deactivate(ctx->net, set, elem);
6544         nft_trans_elem_set(trans) = set;
6545         nft_trans_elem(trans) = *elem;
6546         nft_trans_commit_list_add_tail(ctx->net, trans);
6547
6548         return 0;
6549 err1:
6550         kfree(trans);
6551         return err;
6552 }
6553
6554 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
6555                                     struct nft_set *set,
6556                                     struct nft_set_elem *elem)
6557 {
6558         struct nft_trans *trans;
6559
6560         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6561                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
6562         if (!trans)
6563                 return -ENOMEM;
6564
6565         nft_setelem_data_deactivate(ctx->net, set, elem);
6566         nft_trans_elem_set(trans) = set;
6567         nft_trans_elem(trans) = *elem;
6568         nft_trans_commit_list_add_tail(ctx->net, trans);
6569
6570         return 0;
6571 }
6572
6573 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
6574                                   struct nft_set *set)
6575 {
6576         u8 genmask = nft_genmask_next(ctx->net);
6577         struct nft_set_elem_catchall *catchall;
6578         struct nft_set_elem elem;
6579         struct nft_set_ext *ext;
6580         int ret = 0;
6581
6582         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6583                 ext = nft_set_elem_ext(set, catchall->elem);
6584                 if (!nft_set_elem_active(ext, genmask) ||
6585                     nft_set_elem_mark_busy(ext))
6586                         continue;
6587
6588                 elem.priv = catchall->elem;
6589                 ret = __nft_set_catchall_flush(ctx, set, &elem);
6590                 if (ret < 0)
6591                         break;
6592         }
6593
6594         return ret;
6595 }
6596
6597 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
6598 {
6599         struct nft_set_iter iter = {
6600                 .genmask        = genmask,
6601                 .fn             = nft_setelem_flush,
6602         };
6603
6604         set->ops->walk(ctx, set, &iter);
6605         if (!iter.err)
6606                 iter.err = nft_set_catchall_flush(ctx, set);
6607
6608         return iter.err;
6609 }
6610
6611 static int nf_tables_delsetelem(struct sk_buff *skb,
6612                                 const struct nfnl_info *info,
6613                                 const struct nlattr * const nla[])
6614 {
6615         struct netlink_ext_ack *extack = info->extack;
6616         u8 genmask = nft_genmask_next(info->net);
6617         u8 family = info->nfmsg->nfgen_family;
6618         struct net *net = info->net;
6619         const struct nlattr *attr;
6620         struct nft_table *table;
6621         struct nft_set *set;
6622         struct nft_ctx ctx;
6623         int rem, err = 0;
6624
6625         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6626                                  genmask, NETLINK_CB(skb).portid);
6627         if (IS_ERR(table)) {
6628                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6629                 return PTR_ERR(table);
6630         }
6631
6632         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6633         if (IS_ERR(set))
6634                 return PTR_ERR(set);
6635         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
6636                 return -EBUSY;
6637
6638         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6639
6640         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6641                 return nft_set_flush(&ctx, set, genmask);
6642
6643         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6644                 err = nft_del_setelem(&ctx, set, attr);
6645                 if (err < 0) {
6646                         NL_SET_BAD_ATTR(extack, attr);
6647                         break;
6648                 }
6649         }
6650         return err;
6651 }
6652
6653 void nft_set_gc_batch_release(struct rcu_head *rcu)
6654 {
6655         struct nft_set_gc_batch *gcb;
6656         unsigned int i;
6657
6658         gcb = container_of(rcu, struct nft_set_gc_batch, head.rcu);
6659         for (i = 0; i < gcb->head.cnt; i++)
6660                 nft_set_elem_destroy(gcb->head.set, gcb->elems[i], true);
6661         kfree(gcb);
6662 }
6663
6664 struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
6665                                                 gfp_t gfp)
6666 {
6667         struct nft_set_gc_batch *gcb;
6668
6669         gcb = kzalloc(sizeof(*gcb), gfp);
6670         if (gcb == NULL)
6671                 return gcb;
6672         gcb->head.set = set;
6673         return gcb;
6674 }
6675
6676 /*
6677  * Stateful objects
6678  */
6679
6680 /**
6681  *      nft_register_obj- register nf_tables stateful object type
6682  *      @obj_type: object type
6683  *
6684  *      Registers the object type for use with nf_tables. Returns zero on
6685  *      success or a negative errno code otherwise.
6686  */
6687 int nft_register_obj(struct nft_object_type *obj_type)
6688 {
6689         if (obj_type->type == NFT_OBJECT_UNSPEC)
6690                 return -EINVAL;
6691
6692         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6693         list_add_rcu(&obj_type->list, &nf_tables_objects);
6694         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6695         return 0;
6696 }
6697 EXPORT_SYMBOL_GPL(nft_register_obj);
6698
6699 /**
6700  *      nft_unregister_obj - unregister nf_tables object type
6701  *      @obj_type: object type
6702  *
6703  *      Unregisters the object type for use with nf_tables.
6704  */
6705 void nft_unregister_obj(struct nft_object_type *obj_type)
6706 {
6707         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6708         list_del_rcu(&obj_type->list);
6709         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6710 }
6711 EXPORT_SYMBOL_GPL(nft_unregister_obj);
6712
6713 struct nft_object *nft_obj_lookup(const struct net *net,
6714                                   const struct nft_table *table,
6715                                   const struct nlattr *nla, u32 objtype,
6716                                   u8 genmask)
6717 {
6718         struct nft_object_hash_key k = { .table = table };
6719         char search[NFT_OBJ_MAXNAMELEN];
6720         struct rhlist_head *tmp, *list;
6721         struct nft_object *obj;
6722
6723         nla_strscpy(search, nla, sizeof(search));
6724         k.name = search;
6725
6726         WARN_ON_ONCE(!rcu_read_lock_held() &&
6727                      !lockdep_commit_lock_is_held(net));
6728
6729         rcu_read_lock();
6730         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
6731         if (!list)
6732                 goto out;
6733
6734         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
6735                 if (objtype == obj->ops->type->type &&
6736                     nft_active_genmask(obj, genmask)) {
6737                         rcu_read_unlock();
6738                         return obj;
6739                 }
6740         }
6741 out:
6742         rcu_read_unlock();
6743         return ERR_PTR(-ENOENT);
6744 }
6745 EXPORT_SYMBOL_GPL(nft_obj_lookup);
6746
6747 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
6748                                                   const struct nlattr *nla,
6749                                                   u32 objtype, u8 genmask)
6750 {
6751         struct nft_object *obj;
6752
6753         list_for_each_entry(obj, &table->objects, list) {
6754                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
6755                     objtype == obj->ops->type->type &&
6756                     nft_active_genmask(obj, genmask))
6757                         return obj;
6758         }
6759         return ERR_PTR(-ENOENT);
6760 }
6761
6762 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
6763         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
6764                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
6765         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
6766                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
6767         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
6768         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
6769         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
6770         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
6771                                     .len = NFT_USERDATA_MAXLEN },
6772 };
6773
6774 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
6775                                        const struct nft_object_type *type,
6776                                        const struct nlattr *attr)
6777 {
6778         struct nlattr **tb;
6779         const struct nft_object_ops *ops;
6780         struct nft_object *obj;
6781         int err = -ENOMEM;
6782
6783         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
6784         if (!tb)
6785                 goto err1;
6786
6787         if (attr) {
6788                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
6789                                                   type->policy, NULL);
6790                 if (err < 0)
6791                         goto err2;
6792         } else {
6793                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
6794         }
6795
6796         if (type->select_ops) {
6797                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
6798                 if (IS_ERR(ops)) {
6799                         err = PTR_ERR(ops);
6800                         goto err2;
6801                 }
6802         } else {
6803                 ops = type->ops;
6804         }
6805
6806         err = -ENOMEM;
6807         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
6808         if (!obj)
6809                 goto err2;
6810
6811         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
6812         if (err < 0)
6813                 goto err3;
6814
6815         obj->ops = ops;
6816
6817         kfree(tb);
6818         return obj;
6819 err3:
6820         kfree(obj);
6821 err2:
6822         kfree(tb);
6823 err1:
6824         return ERR_PTR(err);
6825 }
6826
6827 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
6828                            struct nft_object *obj, bool reset)
6829 {
6830         struct nlattr *nest;
6831
6832         nest = nla_nest_start_noflag(skb, attr);
6833         if (!nest)
6834                 goto nla_put_failure;
6835         if (obj->ops->dump(skb, obj, reset) < 0)
6836                 goto nla_put_failure;
6837         nla_nest_end(skb, nest);
6838         return 0;
6839
6840 nla_put_failure:
6841         return -1;
6842 }
6843
6844 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
6845 {
6846         const struct nft_object_type *type;
6847
6848         list_for_each_entry(type, &nf_tables_objects, list) {
6849                 if (objtype == type->type)
6850                         return type;
6851         }
6852         return NULL;
6853 }
6854
6855 static const struct nft_object_type *
6856 nft_obj_type_get(struct net *net, u32 objtype)
6857 {
6858         const struct nft_object_type *type;
6859
6860         type = __nft_obj_type_get(objtype);
6861         if (type != NULL && try_module_get(type->owner))
6862                 return type;
6863
6864         lockdep_nfnl_nft_mutex_not_held();
6865 #ifdef CONFIG_MODULES
6866         if (type == NULL) {
6867                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
6868                         return ERR_PTR(-EAGAIN);
6869         }
6870 #endif
6871         return ERR_PTR(-ENOENT);
6872 }
6873
6874 static int nf_tables_updobj(const struct nft_ctx *ctx,
6875                             const struct nft_object_type *type,
6876                             const struct nlattr *attr,
6877                             struct nft_object *obj)
6878 {
6879         struct nft_object *newobj;
6880         struct nft_trans *trans;
6881         int err = -ENOMEM;
6882
6883         if (!try_module_get(type->owner))
6884                 return -ENOENT;
6885
6886         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
6887                                 sizeof(struct nft_trans_obj));
6888         if (!trans)
6889                 goto err_trans;
6890
6891         newobj = nft_obj_init(ctx, type, attr);
6892         if (IS_ERR(newobj)) {
6893                 err = PTR_ERR(newobj);
6894                 goto err_free_trans;
6895         }
6896
6897         nft_trans_obj(trans) = obj;
6898         nft_trans_obj_update(trans) = true;
6899         nft_trans_obj_newobj(trans) = newobj;
6900         nft_trans_commit_list_add_tail(ctx->net, trans);
6901
6902         return 0;
6903
6904 err_free_trans:
6905         kfree(trans);
6906 err_trans:
6907         module_put(type->owner);
6908         return err;
6909 }
6910
6911 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
6912                             const struct nlattr * const nla[])
6913 {
6914         struct netlink_ext_ack *extack = info->extack;
6915         u8 genmask = nft_genmask_next(info->net);
6916         u8 family = info->nfmsg->nfgen_family;
6917         const struct nft_object_type *type;
6918         struct net *net = info->net;
6919         struct nft_table *table;
6920         struct nft_object *obj;
6921         struct nft_ctx ctx;
6922         u32 objtype;
6923         int err;
6924
6925         if (!nla[NFTA_OBJ_TYPE] ||
6926             !nla[NFTA_OBJ_NAME] ||
6927             !nla[NFTA_OBJ_DATA])
6928                 return -EINVAL;
6929
6930         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
6931                                  NETLINK_CB(skb).portid);
6932         if (IS_ERR(table)) {
6933                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
6934                 return PTR_ERR(table);
6935         }
6936
6937         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
6938         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
6939         if (IS_ERR(obj)) {
6940                 err = PTR_ERR(obj);
6941                 if (err != -ENOENT) {
6942                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
6943                         return err;
6944                 }
6945         } else {
6946                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
6947                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
6948                         return -EEXIST;
6949                 }
6950                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
6951                         return -EOPNOTSUPP;
6952
6953                 type = __nft_obj_type_get(objtype);
6954                 if (WARN_ON_ONCE(!type))
6955                         return -ENOENT;
6956
6957                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6958
6959                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
6960         }
6961
6962         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6963
6964         type = nft_obj_type_get(net, objtype);
6965         if (IS_ERR(type))
6966                 return PTR_ERR(type);
6967
6968         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
6969         if (IS_ERR(obj)) {
6970                 err = PTR_ERR(obj);
6971                 goto err_init;
6972         }
6973         obj->key.table = table;
6974         obj->handle = nf_tables_alloc_handle(table);
6975
6976         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
6977         if (!obj->key.name) {
6978                 err = -ENOMEM;
6979                 goto err_strdup;
6980         }
6981
6982         if (nla[NFTA_OBJ_USERDATA]) {
6983                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL);
6984                 if (obj->udata == NULL)
6985                         goto err_userdata;
6986
6987                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
6988         }
6989
6990         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
6991         if (err < 0)
6992                 goto err_trans;
6993
6994         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
6995                               nft_objname_ht_params);
6996         if (err < 0)
6997                 goto err_obj_ht;
6998
6999         list_add_tail_rcu(&obj->list, &table->objects);
7000         table->use++;
7001         return 0;
7002 err_obj_ht:
7003         /* queued in transaction log */
7004         INIT_LIST_HEAD(&obj->list);
7005         return err;
7006 err_trans:
7007         kfree(obj->udata);
7008 err_userdata:
7009         kfree(obj->key.name);
7010 err_strdup:
7011         if (obj->ops->destroy)
7012                 obj->ops->destroy(&ctx, obj);
7013         kfree(obj);
7014 err_init:
7015         module_put(type->owner);
7016         return err;
7017 }
7018
7019 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7020                                    u32 portid, u32 seq, int event, u32 flags,
7021                                    int family, const struct nft_table *table,
7022                                    struct nft_object *obj, bool reset)
7023 {
7024         struct nlmsghdr *nlh;
7025
7026         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7027         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7028                            NFNETLINK_V0, nft_base_seq(net));
7029         if (!nlh)
7030                 goto nla_put_failure;
7031
7032         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7033             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7034             nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7035             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7036             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset) ||
7037             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7038                          NFTA_OBJ_PAD))
7039                 goto nla_put_failure;
7040
7041         if (obj->udata &&
7042             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7043                 goto nla_put_failure;
7044
7045         nlmsg_end(skb, nlh);
7046         return 0;
7047
7048 nla_put_failure:
7049         nlmsg_trim(skb, nlh);
7050         return -1;
7051 }
7052
7053 struct nft_obj_filter {
7054         char            *table;
7055         u32             type;
7056 };
7057
7058 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7059 {
7060         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7061         const struct nft_table *table;
7062         unsigned int idx = 0, s_idx = cb->args[0];
7063         struct nft_obj_filter *filter = cb->data;
7064         struct net *net = sock_net(skb->sk);
7065         int family = nfmsg->nfgen_family;
7066         struct nftables_pernet *nft_net;
7067         struct nft_object *obj;
7068         bool reset = false;
7069
7070         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7071                 reset = true;
7072
7073         rcu_read_lock();
7074         nft_net = nft_pernet(net);
7075         cb->seq = READ_ONCE(nft_net->base_seq);
7076
7077         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7078                 if (family != NFPROTO_UNSPEC && family != table->family)
7079                         continue;
7080
7081                 list_for_each_entry_rcu(obj, &table->objects, list) {
7082                         if (!nft_is_active(net, obj))
7083                                 goto cont;
7084                         if (idx < s_idx)
7085                                 goto cont;
7086                         if (idx > s_idx)
7087                                 memset(&cb->args[1], 0,
7088                                        sizeof(cb->args) - sizeof(cb->args[0]));
7089                         if (filter && filter->table &&
7090                             strcmp(filter->table, table->name))
7091                                 goto cont;
7092                         if (filter &&
7093                             filter->type != NFT_OBJECT_UNSPEC &&
7094                             obj->ops->type->type != filter->type)
7095                                 goto cont;
7096                         if (reset) {
7097                                 char *buf = kasprintf(GFP_ATOMIC,
7098                                                       "%s:%u",
7099                                                       table->name,
7100                                                       nft_net->base_seq);
7101
7102                                 audit_log_nfcfg(buf,
7103                                                 family,
7104                                                 obj->handle,
7105                                                 AUDIT_NFT_OP_OBJ_RESET,
7106                                                 GFP_ATOMIC);
7107                                 kfree(buf);
7108                         }
7109
7110                         if (nf_tables_fill_obj_info(skb, net, NETLINK_CB(cb->skb).portid,
7111                                                     cb->nlh->nlmsg_seq,
7112                                                     NFT_MSG_NEWOBJ,
7113                                                     NLM_F_MULTI | NLM_F_APPEND,
7114                                                     table->family, table,
7115                                                     obj, reset) < 0)
7116                                 goto done;
7117
7118                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7119 cont:
7120                         idx++;
7121                 }
7122         }
7123 done:
7124         rcu_read_unlock();
7125
7126         cb->args[0] = idx;
7127         return skb->len;
7128 }
7129
7130 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7131 {
7132         const struct nlattr * const *nla = cb->data;
7133         struct nft_obj_filter *filter = NULL;
7134
7135         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
7136                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7137                 if (!filter)
7138                         return -ENOMEM;
7139
7140                 if (nla[NFTA_OBJ_TABLE]) {
7141                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7142                         if (!filter->table) {
7143                                 kfree(filter);
7144                                 return -ENOMEM;
7145                         }
7146                 }
7147
7148                 if (nla[NFTA_OBJ_TYPE])
7149                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7150         }
7151
7152         cb->data = filter;
7153         return 0;
7154 }
7155
7156 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7157 {
7158         struct nft_obj_filter *filter = cb->data;
7159
7160         if (filter) {
7161                 kfree(filter->table);
7162                 kfree(filter);
7163         }
7164
7165         return 0;
7166 }
7167
7168 /* called with rcu_read_lock held */
7169 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7170                             const struct nlattr * const nla[])
7171 {
7172         struct netlink_ext_ack *extack = info->extack;
7173         u8 genmask = nft_genmask_cur(info->net);
7174         u8 family = info->nfmsg->nfgen_family;
7175         const struct nft_table *table;
7176         struct net *net = info->net;
7177         struct nft_object *obj;
7178         struct sk_buff *skb2;
7179         bool reset = false;
7180         u32 objtype;
7181         int err;
7182
7183         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7184                 struct netlink_dump_control c = {
7185                         .start = nf_tables_dump_obj_start,
7186                         .dump = nf_tables_dump_obj,
7187                         .done = nf_tables_dump_obj_done,
7188                         .module = THIS_MODULE,
7189                         .data = (void *)nla,
7190                 };
7191
7192                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7193         }
7194
7195         if (!nla[NFTA_OBJ_NAME] ||
7196             !nla[NFTA_OBJ_TYPE])
7197                 return -EINVAL;
7198
7199         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
7200         if (IS_ERR(table)) {
7201                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7202                 return PTR_ERR(table);
7203         }
7204
7205         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7206         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7207         if (IS_ERR(obj)) {
7208                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7209                 return PTR_ERR(obj);
7210         }
7211
7212         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7213         if (!skb2)
7214                 return -ENOMEM;
7215
7216         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7217                 reset = true;
7218
7219         if (reset) {
7220                 const struct nftables_pernet *nft_net;
7221                 char *buf;
7222
7223                 nft_net = nft_pernet(net);
7224                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
7225
7226                 audit_log_nfcfg(buf,
7227                                 family,
7228                                 obj->handle,
7229                                 AUDIT_NFT_OP_OBJ_RESET,
7230                                 GFP_ATOMIC);
7231                 kfree(buf);
7232         }
7233
7234         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
7235                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
7236                                       family, table, obj, reset);
7237         if (err < 0)
7238                 goto err_fill_obj_info;
7239
7240         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7241
7242 err_fill_obj_info:
7243         kfree_skb(skb2);
7244         return err;
7245 }
7246
7247 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
7248 {
7249         if (obj->ops->destroy)
7250                 obj->ops->destroy(ctx, obj);
7251
7252         module_put(obj->ops->type->owner);
7253         kfree(obj->key.name);
7254         kfree(obj->udata);
7255         kfree(obj);
7256 }
7257
7258 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
7259                             const struct nlattr * const nla[])
7260 {
7261         struct netlink_ext_ack *extack = info->extack;
7262         u8 genmask = nft_genmask_next(info->net);
7263         u8 family = info->nfmsg->nfgen_family;
7264         struct net *net = info->net;
7265         const struct nlattr *attr;
7266         struct nft_table *table;
7267         struct nft_object *obj;
7268         struct nft_ctx ctx;
7269         u32 objtype;
7270
7271         if (!nla[NFTA_OBJ_TYPE] ||
7272             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
7273                 return -EINVAL;
7274
7275         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7276                                  NETLINK_CB(skb).portid);
7277         if (IS_ERR(table)) {
7278                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7279                 return PTR_ERR(table);
7280         }
7281
7282         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7283         if (nla[NFTA_OBJ_HANDLE]) {
7284                 attr = nla[NFTA_OBJ_HANDLE];
7285                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
7286         } else {
7287                 attr = nla[NFTA_OBJ_NAME];
7288                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
7289         }
7290
7291         if (IS_ERR(obj)) {
7292                 NL_SET_BAD_ATTR(extack, attr);
7293                 return PTR_ERR(obj);
7294         }
7295         if (obj->use > 0) {
7296                 NL_SET_BAD_ATTR(extack, attr);
7297                 return -EBUSY;
7298         }
7299
7300         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7301
7302         return nft_delobj(&ctx, obj);
7303 }
7304
7305 void nft_obj_notify(struct net *net, const struct nft_table *table,
7306                     struct nft_object *obj, u32 portid, u32 seq, int event,
7307                     u16 flags, int family, int report, gfp_t gfp)
7308 {
7309         struct nftables_pernet *nft_net = nft_pernet(net);
7310         struct sk_buff *skb;
7311         int err;
7312         char *buf = kasprintf(gfp, "%s:%u",
7313                               table->name, nft_net->base_seq);
7314
7315         audit_log_nfcfg(buf,
7316                         family,
7317                         obj->handle,
7318                         event == NFT_MSG_NEWOBJ ?
7319                                  AUDIT_NFT_OP_OBJ_REGISTER :
7320                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
7321                         gfp);
7322         kfree(buf);
7323
7324         if (!report &&
7325             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7326                 return;
7327
7328         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
7329         if (skb == NULL)
7330                 goto err;
7331
7332         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
7333                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
7334                                       family, table, obj, false);
7335         if (err < 0) {
7336                 kfree_skb(skb);
7337                 goto err;
7338         }
7339
7340         nft_notify_enqueue(skb, report, &nft_net->notify_list);
7341         return;
7342 err:
7343         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
7344 }
7345 EXPORT_SYMBOL_GPL(nft_obj_notify);
7346
7347 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
7348                                  struct nft_object *obj, int event)
7349 {
7350         nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
7351                        ctx->flags, ctx->family, ctx->report, GFP_KERNEL);
7352 }
7353
7354 /*
7355  * Flow tables
7356  */
7357 void nft_register_flowtable_type(struct nf_flowtable_type *type)
7358 {
7359         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7360         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
7361         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7362 }
7363 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
7364
7365 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
7366 {
7367         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7368         list_del_rcu(&type->list);
7369         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7370 }
7371 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
7372
7373 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
7374         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
7375                                             .len = NFT_NAME_MAXLEN - 1 },
7376         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
7377                                             .len = NFT_NAME_MAXLEN - 1 },
7378         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
7379         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
7380         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
7381 };
7382
7383 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
7384                                            const struct nlattr *nla, u8 genmask)
7385 {
7386         struct nft_flowtable *flowtable;
7387
7388         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
7389                 if (!nla_strcmp(nla, flowtable->name) &&
7390                     nft_active_genmask(flowtable, genmask))
7391                         return flowtable;
7392         }
7393         return ERR_PTR(-ENOENT);
7394 }
7395 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
7396
7397 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
7398                                     struct nft_flowtable *flowtable,
7399                                     enum nft_trans_phase phase)
7400 {
7401         switch (phase) {
7402         case NFT_TRANS_PREPARE:
7403         case NFT_TRANS_ABORT:
7404         case NFT_TRANS_RELEASE:
7405                 flowtable->use--;
7406                 fallthrough;
7407         default:
7408                 return;
7409         }
7410 }
7411 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
7412
7413 static struct nft_flowtable *
7414 nft_flowtable_lookup_byhandle(const struct nft_table *table,
7415                               const struct nlattr *nla, u8 genmask)
7416 {
7417        struct nft_flowtable *flowtable;
7418
7419        list_for_each_entry(flowtable, &table->flowtables, list) {
7420                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
7421                    nft_active_genmask(flowtable, genmask))
7422                        return flowtable;
7423        }
7424        return ERR_PTR(-ENOENT);
7425 }
7426
7427 struct nft_flowtable_hook {
7428         u32                     num;
7429         int                     priority;
7430         struct list_head        list;
7431 };
7432
7433 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
7434         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
7435         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
7436         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
7437 };
7438
7439 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
7440                                     const struct nlattr *attr,
7441                                     struct nft_flowtable_hook *flowtable_hook,
7442                                     struct nft_flowtable *flowtable, bool add)
7443 {
7444         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
7445         struct nft_hook *hook;
7446         int hooknum, priority;
7447         int err;
7448
7449         INIT_LIST_HEAD(&flowtable_hook->list);
7450
7451         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX, attr,
7452                                           nft_flowtable_hook_policy, NULL);
7453         if (err < 0)
7454                 return err;
7455
7456         if (add) {
7457                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
7458                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY])
7459                         return -EINVAL;
7460
7461                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7462                 if (hooknum != NF_NETDEV_INGRESS)
7463                         return -EOPNOTSUPP;
7464
7465                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7466
7467                 flowtable_hook->priority        = priority;
7468                 flowtable_hook->num             = hooknum;
7469         } else {
7470                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
7471                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7472                         if (hooknum != flowtable->hooknum)
7473                                 return -EOPNOTSUPP;
7474                 }
7475
7476                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
7477                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7478                         if (priority != flowtable->data.priority)
7479                                 return -EOPNOTSUPP;
7480                 }
7481
7482                 flowtable_hook->priority        = flowtable->data.priority;
7483                 flowtable_hook->num             = flowtable->hooknum;
7484         }
7485
7486         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
7487                 err = nf_tables_parse_netdev_hooks(ctx->net,
7488                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
7489                                                    &flowtable_hook->list);
7490                 if (err < 0)
7491                         return err;
7492         }
7493
7494         list_for_each_entry(hook, &flowtable_hook->list, list) {
7495                 hook->ops.pf            = NFPROTO_NETDEV;
7496                 hook->ops.hooknum       = flowtable_hook->num;
7497                 hook->ops.priority      = flowtable_hook->priority;
7498                 hook->ops.priv          = &flowtable->data;
7499                 hook->ops.hook          = flowtable->data.type->hook;
7500         }
7501
7502         return err;
7503 }
7504
7505 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
7506 {
7507         const struct nf_flowtable_type *type;
7508
7509         list_for_each_entry(type, &nf_tables_flowtables, list) {
7510                 if (family == type->family)
7511                         return type;
7512         }
7513         return NULL;
7514 }
7515
7516 static const struct nf_flowtable_type *
7517 nft_flowtable_type_get(struct net *net, u8 family)
7518 {
7519         const struct nf_flowtable_type *type;
7520
7521         type = __nft_flowtable_type_get(family);
7522         if (type != NULL && try_module_get(type->owner))
7523                 return type;
7524
7525         lockdep_nfnl_nft_mutex_not_held();
7526 #ifdef CONFIG_MODULES
7527         if (type == NULL) {
7528                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
7529                         return ERR_PTR(-EAGAIN);
7530         }
7531 #endif
7532         return ERR_PTR(-ENOENT);
7533 }
7534
7535 /* Only called from error and netdev event paths. */
7536 static void nft_unregister_flowtable_hook(struct net *net,
7537                                           struct nft_flowtable *flowtable,
7538                                           struct nft_hook *hook)
7539 {
7540         nf_unregister_net_hook(net, &hook->ops);
7541         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
7542                                     FLOW_BLOCK_UNBIND);
7543 }
7544
7545 static void __nft_unregister_flowtable_net_hooks(struct net *net,
7546                                                  struct list_head *hook_list,
7547                                                  bool release_netdev)
7548 {
7549         struct nft_hook *hook, *next;
7550
7551         list_for_each_entry_safe(hook, next, hook_list, list) {
7552                 nf_unregister_net_hook(net, &hook->ops);
7553                 if (release_netdev) {
7554                         list_del(&hook->list);
7555                         kfree_rcu(hook, rcu);
7556                 }
7557         }
7558 }
7559
7560 static void nft_unregister_flowtable_net_hooks(struct net *net,
7561                                                struct list_head *hook_list)
7562 {
7563         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
7564 }
7565
7566 static int nft_register_flowtable_net_hooks(struct net *net,
7567                                             struct nft_table *table,
7568                                             struct list_head *hook_list,
7569                                             struct nft_flowtable *flowtable)
7570 {
7571         struct nft_hook *hook, *hook2, *next;
7572         struct nft_flowtable *ft;
7573         int err, i = 0;
7574
7575         list_for_each_entry(hook, hook_list, list) {
7576                 list_for_each_entry(ft, &table->flowtables, list) {
7577                         if (!nft_is_active_next(net, ft))
7578                                 continue;
7579
7580                         list_for_each_entry(hook2, &ft->hook_list, list) {
7581                                 if (hook->ops.dev == hook2->ops.dev &&
7582                                     hook->ops.pf == hook2->ops.pf) {
7583                                         err = -EEXIST;
7584                                         goto err_unregister_net_hooks;
7585                                 }
7586                         }
7587                 }
7588
7589                 err = flowtable->data.type->setup(&flowtable->data,
7590                                                   hook->ops.dev,
7591                                                   FLOW_BLOCK_BIND);
7592                 if (err < 0)
7593                         goto err_unregister_net_hooks;
7594
7595                 err = nf_register_net_hook(net, &hook->ops);
7596                 if (err < 0) {
7597                         flowtable->data.type->setup(&flowtable->data,
7598                                                     hook->ops.dev,
7599                                                     FLOW_BLOCK_UNBIND);
7600                         goto err_unregister_net_hooks;
7601                 }
7602
7603                 i++;
7604         }
7605
7606         return 0;
7607
7608 err_unregister_net_hooks:
7609         list_for_each_entry_safe(hook, next, hook_list, list) {
7610                 if (i-- <= 0)
7611                         break;
7612
7613                 nft_unregister_flowtable_hook(net, flowtable, hook);
7614                 list_del_rcu(&hook->list);
7615                 kfree_rcu(hook, rcu);
7616         }
7617
7618         return err;
7619 }
7620
7621 static void nft_flowtable_hooks_destroy(struct list_head *hook_list)
7622 {
7623         struct nft_hook *hook, *next;
7624
7625         list_for_each_entry_safe(hook, next, hook_list, list) {
7626                 list_del_rcu(&hook->list);
7627                 kfree_rcu(hook, rcu);
7628         }
7629 }
7630
7631 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
7632                                 struct nft_flowtable *flowtable)
7633 {
7634         const struct nlattr * const *nla = ctx->nla;
7635         struct nft_flowtable_hook flowtable_hook;
7636         struct nft_hook *hook, *next;
7637         struct nft_trans *trans;
7638         bool unregister = false;
7639         u32 flags;
7640         int err;
7641
7642         err = nft_flowtable_parse_hook(ctx, nla[NFTA_FLOWTABLE_HOOK],
7643                                        &flowtable_hook, flowtable, false);
7644         if (err < 0)
7645                 return err;
7646
7647         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
7648                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
7649                         list_del(&hook->list);
7650                         kfree(hook);
7651                 }
7652         }
7653
7654         if (nla[NFTA_FLOWTABLE_FLAGS]) {
7655                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
7656                 if (flags & ~NFT_FLOWTABLE_MASK) {
7657                         err = -EOPNOTSUPP;
7658                         goto err_flowtable_update_hook;
7659                 }
7660                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
7661                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
7662                         err = -EOPNOTSUPP;
7663                         goto err_flowtable_update_hook;
7664                 }
7665         } else {
7666                 flags = flowtable->data.flags;
7667         }
7668
7669         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
7670                                                &flowtable_hook.list, flowtable);
7671         if (err < 0)
7672                 goto err_flowtable_update_hook;
7673
7674         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
7675                                 sizeof(struct nft_trans_flowtable));
7676         if (!trans) {
7677                 unregister = true;
7678                 err = -ENOMEM;
7679                 goto err_flowtable_update_hook;
7680         }
7681
7682         nft_trans_flowtable_flags(trans) = flags;
7683         nft_trans_flowtable(trans) = flowtable;
7684         nft_trans_flowtable_update(trans) = true;
7685         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
7686         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
7687
7688         nft_trans_commit_list_add_tail(ctx->net, trans);
7689
7690         return 0;
7691
7692 err_flowtable_update_hook:
7693         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
7694                 if (unregister)
7695                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
7696                 list_del_rcu(&hook->list);
7697                 kfree_rcu(hook, rcu);
7698         }
7699
7700         return err;
7701
7702 }
7703
7704 static int nf_tables_newflowtable(struct sk_buff *skb,
7705                                   const struct nfnl_info *info,
7706                                   const struct nlattr * const nla[])
7707 {
7708         struct netlink_ext_ack *extack = info->extack;
7709         struct nft_flowtable_hook flowtable_hook;
7710         u8 genmask = nft_genmask_next(info->net);
7711         u8 family = info->nfmsg->nfgen_family;
7712         const struct nf_flowtable_type *type;
7713         struct nft_flowtable *flowtable;
7714         struct nft_hook *hook, *next;
7715         struct net *net = info->net;
7716         struct nft_table *table;
7717         struct nft_ctx ctx;
7718         int err;
7719
7720         if (!nla[NFTA_FLOWTABLE_TABLE] ||
7721             !nla[NFTA_FLOWTABLE_NAME] ||
7722             !nla[NFTA_FLOWTABLE_HOOK])
7723                 return -EINVAL;
7724
7725         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
7726                                  genmask, NETLINK_CB(skb).portid);
7727         if (IS_ERR(table)) {
7728                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
7729                 return PTR_ERR(table);
7730         }
7731
7732         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
7733                                          genmask);
7734         if (IS_ERR(flowtable)) {
7735                 err = PTR_ERR(flowtable);
7736                 if (err != -ENOENT) {
7737                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
7738                         return err;
7739                 }
7740         } else {
7741                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7742                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
7743                         return -EEXIST;
7744                 }
7745
7746                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7747
7748                 return nft_flowtable_update(&ctx, info->nlh, flowtable);
7749         }
7750
7751         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7752
7753         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
7754         if (!flowtable)
7755                 return -ENOMEM;
7756
7757         flowtable->table = table;
7758         flowtable->handle = nf_tables_alloc_handle(table);
7759         INIT_LIST_HEAD(&flowtable->hook_list);
7760
7761         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
7762         if (!flowtable->name) {
7763                 err = -ENOMEM;
7764                 goto err1;
7765         }
7766
7767         type = nft_flowtable_type_get(net, family);
7768         if (IS_ERR(type)) {
7769                 err = PTR_ERR(type);
7770                 goto err2;
7771         }
7772
7773         if (nla[NFTA_FLOWTABLE_FLAGS]) {
7774                 flowtable->data.flags =
7775                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
7776                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
7777                         err = -EOPNOTSUPP;
7778                         goto err3;
7779                 }
7780         }
7781
7782         write_pnet(&flowtable->data.net, net);
7783         flowtable->data.type = type;
7784         err = type->init(&flowtable->data);
7785         if (err < 0)
7786                 goto err3;
7787
7788         err = nft_flowtable_parse_hook(&ctx, nla[NFTA_FLOWTABLE_HOOK],
7789                                        &flowtable_hook, flowtable, true);
7790         if (err < 0)
7791                 goto err4;
7792
7793         list_splice(&flowtable_hook.list, &flowtable->hook_list);
7794         flowtable->data.priority = flowtable_hook.priority;
7795         flowtable->hooknum = flowtable_hook.num;
7796
7797         err = nft_register_flowtable_net_hooks(ctx.net, table,
7798                                                &flowtable->hook_list,
7799                                                flowtable);
7800         if (err < 0) {
7801                 nft_flowtable_hooks_destroy(&flowtable->hook_list);
7802                 goto err4;
7803         }
7804
7805         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
7806         if (err < 0)
7807                 goto err5;
7808
7809         list_add_tail_rcu(&flowtable->list, &table->flowtables);
7810         table->use++;
7811
7812         return 0;
7813 err5:
7814         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
7815                 nft_unregister_flowtable_hook(net, flowtable, hook);
7816                 list_del_rcu(&hook->list);
7817                 kfree_rcu(hook, rcu);
7818         }
7819 err4:
7820         flowtable->data.type->free(&flowtable->data);
7821 err3:
7822         module_put(type->owner);
7823 err2:
7824         kfree(flowtable->name);
7825 err1:
7826         kfree(flowtable);
7827         return err;
7828 }
7829
7830 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
7831 {
7832         struct nft_hook *this, *next;
7833
7834         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
7835                 list_del(&this->list);
7836                 kfree(this);
7837         }
7838 }
7839
7840 static int nft_delflowtable_hook(struct nft_ctx *ctx,
7841                                  struct nft_flowtable *flowtable)
7842 {
7843         const struct nlattr * const *nla = ctx->nla;
7844         struct nft_flowtable_hook flowtable_hook;
7845         LIST_HEAD(flowtable_del_list);
7846         struct nft_hook *this, *hook;
7847         struct nft_trans *trans;
7848         int err;
7849
7850         err = nft_flowtable_parse_hook(ctx, nla[NFTA_FLOWTABLE_HOOK],
7851                                        &flowtable_hook, flowtable, false);
7852         if (err < 0)
7853                 return err;
7854
7855         list_for_each_entry(this, &flowtable_hook.list, list) {
7856                 hook = nft_hook_list_find(&flowtable->hook_list, this);
7857                 if (!hook) {
7858                         err = -ENOENT;
7859                         goto err_flowtable_del_hook;
7860                 }
7861                 list_move(&hook->list, &flowtable_del_list);
7862         }
7863
7864         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
7865                                 sizeof(struct nft_trans_flowtable));
7866         if (!trans) {
7867                 err = -ENOMEM;
7868                 goto err_flowtable_del_hook;
7869         }
7870
7871         nft_trans_flowtable(trans) = flowtable;
7872         nft_trans_flowtable_update(trans) = true;
7873         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
7874         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
7875         nft_flowtable_hook_release(&flowtable_hook);
7876
7877         nft_trans_commit_list_add_tail(ctx->net, trans);
7878
7879         return 0;
7880
7881 err_flowtable_del_hook:
7882         list_splice(&flowtable_del_list, &flowtable->hook_list);
7883         nft_flowtable_hook_release(&flowtable_hook);
7884
7885         return err;
7886 }
7887
7888 static int nf_tables_delflowtable(struct sk_buff *skb,
7889                                   const struct nfnl_info *info,
7890                                   const struct nlattr * const nla[])
7891 {
7892         struct netlink_ext_ack *extack = info->extack;
7893         u8 genmask = nft_genmask_next(info->net);
7894         u8 family = info->nfmsg->nfgen_family;
7895         struct nft_flowtable *flowtable;
7896         struct net *net = info->net;
7897         const struct nlattr *attr;
7898         struct nft_table *table;
7899         struct nft_ctx ctx;
7900
7901         if (!nla[NFTA_FLOWTABLE_TABLE] ||
7902             (!nla[NFTA_FLOWTABLE_NAME] &&
7903              !nla[NFTA_FLOWTABLE_HANDLE]))
7904                 return -EINVAL;
7905
7906         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
7907                                  genmask, NETLINK_CB(skb).portid);
7908         if (IS_ERR(table)) {
7909                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
7910                 return PTR_ERR(table);
7911         }
7912
7913         if (nla[NFTA_FLOWTABLE_HANDLE]) {
7914                 attr = nla[NFTA_FLOWTABLE_HANDLE];
7915                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
7916         } else {
7917                 attr = nla[NFTA_FLOWTABLE_NAME];
7918                 flowtable = nft_flowtable_lookup(table, attr, genmask);
7919         }
7920
7921         if (IS_ERR(flowtable)) {
7922                 NL_SET_BAD_ATTR(extack, attr);
7923                 return PTR_ERR(flowtable);
7924         }
7925
7926         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7927
7928         if (nla[NFTA_FLOWTABLE_HOOK])
7929                 return nft_delflowtable_hook(&ctx, flowtable);
7930
7931         if (flowtable->use > 0) {
7932                 NL_SET_BAD_ATTR(extack, attr);
7933                 return -EBUSY;
7934         }
7935
7936         return nft_delflowtable(&ctx, flowtable);
7937 }
7938
7939 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
7940                                          u32 portid, u32 seq, int event,
7941                                          u32 flags, int family,
7942                                          struct nft_flowtable *flowtable,
7943                                          struct list_head *hook_list)
7944 {
7945         struct nlattr *nest, *nest_devs;
7946         struct nft_hook *hook;
7947         struct nlmsghdr *nlh;
7948
7949         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7950         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7951                            NFNETLINK_V0, nft_base_seq(net));
7952         if (!nlh)
7953                 goto nla_put_failure;
7954
7955         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
7956             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
7957             nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
7958             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
7959                          NFTA_FLOWTABLE_PAD) ||
7960             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
7961                 goto nla_put_failure;
7962
7963         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
7964         if (!nest)
7965                 goto nla_put_failure;
7966         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
7967             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
7968                 goto nla_put_failure;
7969
7970         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
7971         if (!nest_devs)
7972                 goto nla_put_failure;
7973
7974         list_for_each_entry_rcu(hook, hook_list, list) {
7975                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
7976                         goto nla_put_failure;
7977         }
7978         nla_nest_end(skb, nest_devs);
7979         nla_nest_end(skb, nest);
7980
7981         nlmsg_end(skb, nlh);
7982         return 0;
7983
7984 nla_put_failure:
7985         nlmsg_trim(skb, nlh);
7986         return -1;
7987 }
7988
7989 struct nft_flowtable_filter {
7990         char            *table;
7991 };
7992
7993 static int nf_tables_dump_flowtable(struct sk_buff *skb,
7994                                     struct netlink_callback *cb)
7995 {
7996         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7997         struct nft_flowtable_filter *filter = cb->data;
7998         unsigned int idx = 0, s_idx = cb->args[0];
7999         struct net *net = sock_net(skb->sk);
8000         int family = nfmsg->nfgen_family;
8001         struct nft_flowtable *flowtable;
8002         struct nftables_pernet *nft_net;
8003         const struct nft_table *table;
8004
8005         rcu_read_lock();
8006         nft_net = nft_pernet(net);
8007         cb->seq = READ_ONCE(nft_net->base_seq);
8008
8009         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8010                 if (family != NFPROTO_UNSPEC && family != table->family)
8011                         continue;
8012
8013                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8014                         if (!nft_is_active(net, flowtable))
8015                                 goto cont;
8016                         if (idx < s_idx)
8017                                 goto cont;
8018                         if (idx > s_idx)
8019                                 memset(&cb->args[1], 0,
8020                                        sizeof(cb->args) - sizeof(cb->args[0]));
8021                         if (filter && filter->table &&
8022                             strcmp(filter->table, table->name))
8023                                 goto cont;
8024
8025                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8026                                                           cb->nlh->nlmsg_seq,
8027                                                           NFT_MSG_NEWFLOWTABLE,
8028                                                           NLM_F_MULTI | NLM_F_APPEND,
8029                                                           table->family,
8030                                                           flowtable,
8031                                                           &flowtable->hook_list) < 0)
8032                                 goto done;
8033
8034                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8035 cont:
8036                         idx++;
8037                 }
8038         }
8039 done:
8040         rcu_read_unlock();
8041
8042         cb->args[0] = idx;
8043         return skb->len;
8044 }
8045
8046 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8047 {
8048         const struct nlattr * const *nla = cb->data;
8049         struct nft_flowtable_filter *filter = NULL;
8050
8051         if (nla[NFTA_FLOWTABLE_TABLE]) {
8052                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8053                 if (!filter)
8054                         return -ENOMEM;
8055
8056                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8057                                            GFP_ATOMIC);
8058                 if (!filter->table) {
8059                         kfree(filter);
8060                         return -ENOMEM;
8061                 }
8062         }
8063
8064         cb->data = filter;
8065         return 0;
8066 }
8067
8068 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8069 {
8070         struct nft_flowtable_filter *filter = cb->data;
8071
8072         if (!filter)
8073                 return 0;
8074
8075         kfree(filter->table);
8076         kfree(filter);
8077
8078         return 0;
8079 }
8080
8081 /* called with rcu_read_lock held */
8082 static int nf_tables_getflowtable(struct sk_buff *skb,
8083                                   const struct nfnl_info *info,
8084                                   const struct nlattr * const nla[])
8085 {
8086         u8 genmask = nft_genmask_cur(info->net);
8087         u8 family = info->nfmsg->nfgen_family;
8088         struct nft_flowtable *flowtable;
8089         const struct nft_table *table;
8090         struct net *net = info->net;
8091         struct sk_buff *skb2;
8092         int err;
8093
8094         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8095                 struct netlink_dump_control c = {
8096                         .start = nf_tables_dump_flowtable_start,
8097                         .dump = nf_tables_dump_flowtable,
8098                         .done = nf_tables_dump_flowtable_done,
8099                         .module = THIS_MODULE,
8100                         .data = (void *)nla,
8101                 };
8102
8103                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8104         }
8105
8106         if (!nla[NFTA_FLOWTABLE_NAME])
8107                 return -EINVAL;
8108
8109         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8110                                  genmask, 0);
8111         if (IS_ERR(table))
8112                 return PTR_ERR(table);
8113
8114         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8115                                          genmask);
8116         if (IS_ERR(flowtable))
8117                 return PTR_ERR(flowtable);
8118
8119         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8120         if (!skb2)
8121                 return -ENOMEM;
8122
8123         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8124                                             info->nlh->nlmsg_seq,
8125                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8126                                             flowtable, &flowtable->hook_list);
8127         if (err < 0)
8128                 goto err_fill_flowtable_info;
8129
8130         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8131
8132 err_fill_flowtable_info:
8133         kfree_skb(skb2);
8134         return err;
8135 }
8136
8137 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
8138                                        struct nft_flowtable *flowtable,
8139                                        struct list_head *hook_list,
8140                                        int event)
8141 {
8142         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
8143         struct sk_buff *skb;
8144         u16 flags = 0;
8145         int err;
8146
8147         if (!ctx->report &&
8148             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
8149                 return;
8150
8151         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8152         if (skb == NULL)
8153                 goto err;
8154
8155         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
8156                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
8157
8158         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
8159                                             ctx->seq, event, flags,
8160                                             ctx->family, flowtable, hook_list);
8161         if (err < 0) {
8162                 kfree_skb(skb);
8163                 goto err;
8164         }
8165
8166         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
8167         return;
8168 err:
8169         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
8170 }
8171
8172 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
8173 {
8174         struct nft_hook *hook, *next;
8175
8176         flowtable->data.type->free(&flowtable->data);
8177         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8178                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8179                                             FLOW_BLOCK_UNBIND);
8180                 list_del_rcu(&hook->list);
8181                 kfree(hook);
8182         }
8183         kfree(flowtable->name);
8184         module_put(flowtable->data.type->owner);
8185         kfree(flowtable);
8186 }
8187
8188 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
8189                                    u32 portid, u32 seq)
8190 {
8191         struct nftables_pernet *nft_net = nft_pernet(net);
8192         struct nlmsghdr *nlh;
8193         char buf[TASK_COMM_LEN];
8194         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
8195
8196         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
8197                            NFNETLINK_V0, nft_base_seq(net));
8198         if (!nlh)
8199                 goto nla_put_failure;
8200
8201         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
8202             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
8203             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
8204                 goto nla_put_failure;
8205
8206         nlmsg_end(skb, nlh);
8207         return 0;
8208
8209 nla_put_failure:
8210         nlmsg_trim(skb, nlh);
8211         return -EMSGSIZE;
8212 }
8213
8214 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
8215                                 struct nft_flowtable *flowtable)
8216 {
8217         struct nft_hook *hook;
8218
8219         list_for_each_entry(hook, &flowtable->hook_list, list) {
8220                 if (hook->ops.dev != dev)
8221                         continue;
8222
8223                 /* flow_offload_netdev_event() cleans up entries for us. */
8224                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
8225                 list_del_rcu(&hook->list);
8226                 kfree_rcu(hook, rcu);
8227                 break;
8228         }
8229 }
8230
8231 static int nf_tables_flowtable_event(struct notifier_block *this,
8232                                      unsigned long event, void *ptr)
8233 {
8234         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
8235         struct nft_flowtable *flowtable;
8236         struct nftables_pernet *nft_net;
8237         struct nft_table *table;
8238         struct net *net;
8239
8240         if (event != NETDEV_UNREGISTER)
8241                 return 0;
8242
8243         net = dev_net(dev);
8244         nft_net = nft_pernet(net);
8245         mutex_lock(&nft_net->commit_mutex);
8246         list_for_each_entry(table, &nft_net->tables, list) {
8247                 list_for_each_entry(flowtable, &table->flowtables, list) {
8248                         nft_flowtable_event(event, dev, flowtable);
8249                 }
8250         }
8251         mutex_unlock(&nft_net->commit_mutex);
8252
8253         return NOTIFY_DONE;
8254 }
8255
8256 static struct notifier_block nf_tables_flowtable_notifier = {
8257         .notifier_call  = nf_tables_flowtable_event,
8258 };
8259
8260 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
8261                                  int event)
8262 {
8263         struct nlmsghdr *nlh = nlmsg_hdr(skb);
8264         struct sk_buff *skb2;
8265         int err;
8266
8267         if (!nlmsg_report(nlh) &&
8268             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8269                 return;
8270
8271         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8272         if (skb2 == NULL)
8273                 goto err;
8274
8275         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
8276                                       nlh->nlmsg_seq);
8277         if (err < 0) {
8278                 kfree_skb(skb2);
8279                 goto err;
8280         }
8281
8282         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8283                        nlmsg_report(nlh), GFP_KERNEL);
8284         return;
8285 err:
8286         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
8287                           -ENOBUFS);
8288 }
8289
8290 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
8291                             const struct nlattr * const nla[])
8292 {
8293         struct sk_buff *skb2;
8294         int err;
8295
8296         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8297         if (skb2 == NULL)
8298                 return -ENOMEM;
8299
8300         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
8301                                       info->nlh->nlmsg_seq);
8302         if (err < 0)
8303                 goto err_fill_gen_info;
8304
8305         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
8306
8307 err_fill_gen_info:
8308         kfree_skb(skb2);
8309         return err;
8310 }
8311
8312 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
8313         [NFT_MSG_NEWTABLE] = {
8314                 .call           = nf_tables_newtable,
8315                 .type           = NFNL_CB_BATCH,
8316                 .attr_count     = NFTA_TABLE_MAX,
8317                 .policy         = nft_table_policy,
8318         },
8319         [NFT_MSG_GETTABLE] = {
8320                 .call           = nf_tables_gettable,
8321                 .type           = NFNL_CB_RCU,
8322                 .attr_count     = NFTA_TABLE_MAX,
8323                 .policy         = nft_table_policy,
8324         },
8325         [NFT_MSG_DELTABLE] = {
8326                 .call           = nf_tables_deltable,
8327                 .type           = NFNL_CB_BATCH,
8328                 .attr_count     = NFTA_TABLE_MAX,
8329                 .policy         = nft_table_policy,
8330         },
8331         [NFT_MSG_NEWCHAIN] = {
8332                 .call           = nf_tables_newchain,
8333                 .type           = NFNL_CB_BATCH,
8334                 .attr_count     = NFTA_CHAIN_MAX,
8335                 .policy         = nft_chain_policy,
8336         },
8337         [NFT_MSG_GETCHAIN] = {
8338                 .call           = nf_tables_getchain,
8339                 .type           = NFNL_CB_RCU,
8340                 .attr_count     = NFTA_CHAIN_MAX,
8341                 .policy         = nft_chain_policy,
8342         },
8343         [NFT_MSG_DELCHAIN] = {
8344                 .call           = nf_tables_delchain,
8345                 .type           = NFNL_CB_BATCH,
8346                 .attr_count     = NFTA_CHAIN_MAX,
8347                 .policy         = nft_chain_policy,
8348         },
8349         [NFT_MSG_NEWRULE] = {
8350                 .call           = nf_tables_newrule,
8351                 .type           = NFNL_CB_BATCH,
8352                 .attr_count     = NFTA_RULE_MAX,
8353                 .policy         = nft_rule_policy,
8354         },
8355         [NFT_MSG_GETRULE] = {
8356                 .call           = nf_tables_getrule,
8357                 .type           = NFNL_CB_RCU,
8358                 .attr_count     = NFTA_RULE_MAX,
8359                 .policy         = nft_rule_policy,
8360         },
8361         [NFT_MSG_DELRULE] = {
8362                 .call           = nf_tables_delrule,
8363                 .type           = NFNL_CB_BATCH,
8364                 .attr_count     = NFTA_RULE_MAX,
8365                 .policy         = nft_rule_policy,
8366         },
8367         [NFT_MSG_NEWSET] = {
8368                 .call           = nf_tables_newset,
8369                 .type           = NFNL_CB_BATCH,
8370                 .attr_count     = NFTA_SET_MAX,
8371                 .policy         = nft_set_policy,
8372         },
8373         [NFT_MSG_GETSET] = {
8374                 .call           = nf_tables_getset,
8375                 .type           = NFNL_CB_RCU,
8376                 .attr_count     = NFTA_SET_MAX,
8377                 .policy         = nft_set_policy,
8378         },
8379         [NFT_MSG_DELSET] = {
8380                 .call           = nf_tables_delset,
8381                 .type           = NFNL_CB_BATCH,
8382                 .attr_count     = NFTA_SET_MAX,
8383                 .policy         = nft_set_policy,
8384         },
8385         [NFT_MSG_NEWSETELEM] = {
8386                 .call           = nf_tables_newsetelem,
8387                 .type           = NFNL_CB_BATCH,
8388                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8389                 .policy         = nft_set_elem_list_policy,
8390         },
8391         [NFT_MSG_GETSETELEM] = {
8392                 .call           = nf_tables_getsetelem,
8393                 .type           = NFNL_CB_RCU,
8394                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8395                 .policy         = nft_set_elem_list_policy,
8396         },
8397         [NFT_MSG_DELSETELEM] = {
8398                 .call           = nf_tables_delsetelem,
8399                 .type           = NFNL_CB_BATCH,
8400                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
8401                 .policy         = nft_set_elem_list_policy,
8402         },
8403         [NFT_MSG_GETGEN] = {
8404                 .call           = nf_tables_getgen,
8405                 .type           = NFNL_CB_RCU,
8406         },
8407         [NFT_MSG_NEWOBJ] = {
8408                 .call           = nf_tables_newobj,
8409                 .type           = NFNL_CB_BATCH,
8410                 .attr_count     = NFTA_OBJ_MAX,
8411                 .policy         = nft_obj_policy,
8412         },
8413         [NFT_MSG_GETOBJ] = {
8414                 .call           = nf_tables_getobj,
8415                 .type           = NFNL_CB_RCU,
8416                 .attr_count     = NFTA_OBJ_MAX,
8417                 .policy         = nft_obj_policy,
8418         },
8419         [NFT_MSG_DELOBJ] = {
8420                 .call           = nf_tables_delobj,
8421                 .type           = NFNL_CB_BATCH,
8422                 .attr_count     = NFTA_OBJ_MAX,
8423                 .policy         = nft_obj_policy,
8424         },
8425         [NFT_MSG_GETOBJ_RESET] = {
8426                 .call           = nf_tables_getobj,
8427                 .type           = NFNL_CB_RCU,
8428                 .attr_count     = NFTA_OBJ_MAX,
8429                 .policy         = nft_obj_policy,
8430         },
8431         [NFT_MSG_NEWFLOWTABLE] = {
8432                 .call           = nf_tables_newflowtable,
8433                 .type           = NFNL_CB_BATCH,
8434                 .attr_count     = NFTA_FLOWTABLE_MAX,
8435                 .policy         = nft_flowtable_policy,
8436         },
8437         [NFT_MSG_GETFLOWTABLE] = {
8438                 .call           = nf_tables_getflowtable,
8439                 .type           = NFNL_CB_RCU,
8440                 .attr_count     = NFTA_FLOWTABLE_MAX,
8441                 .policy         = nft_flowtable_policy,
8442         },
8443         [NFT_MSG_DELFLOWTABLE] = {
8444                 .call           = nf_tables_delflowtable,
8445                 .type           = NFNL_CB_BATCH,
8446                 .attr_count     = NFTA_FLOWTABLE_MAX,
8447                 .policy         = nft_flowtable_policy,
8448         },
8449 };
8450
8451 static int nf_tables_validate(struct net *net)
8452 {
8453         struct nftables_pernet *nft_net = nft_pernet(net);
8454         struct nft_table *table;
8455
8456         switch (nft_net->validate_state) {
8457         case NFT_VALIDATE_SKIP:
8458                 break;
8459         case NFT_VALIDATE_NEED:
8460                 nft_validate_state_update(net, NFT_VALIDATE_DO);
8461                 fallthrough;
8462         case NFT_VALIDATE_DO:
8463                 list_for_each_entry(table, &nft_net->tables, list) {
8464                         if (nft_table_validate(net, table) < 0)
8465                                 return -EAGAIN;
8466                 }
8467                 break;
8468         }
8469
8470         return 0;
8471 }
8472
8473 /* a drop policy has to be deferred until all rules have been activated,
8474  * otherwise a large ruleset that contains a drop-policy base chain will
8475  * cause all packets to get dropped until the full transaction has been
8476  * processed.
8477  *
8478  * We defer the drop policy until the transaction has been finalized.
8479  */
8480 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
8481 {
8482         struct nft_base_chain *basechain;
8483
8484         if (nft_trans_chain_policy(trans) != NF_DROP)
8485                 return;
8486
8487         if (!nft_is_base_chain(trans->ctx.chain))
8488                 return;
8489
8490         basechain = nft_base_chain(trans->ctx.chain);
8491         basechain->policy = NF_DROP;
8492 }
8493
8494 static void nft_chain_commit_update(struct nft_trans *trans)
8495 {
8496         struct nft_base_chain *basechain;
8497
8498         if (nft_trans_chain_name(trans)) {
8499                 rhltable_remove(&trans->ctx.table->chains_ht,
8500                                 &trans->ctx.chain->rhlhead,
8501                                 nft_chain_ht_params);
8502                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
8503                 rhltable_insert_key(&trans->ctx.table->chains_ht,
8504                                     trans->ctx.chain->name,
8505                                     &trans->ctx.chain->rhlhead,
8506                                     nft_chain_ht_params);
8507         }
8508
8509         if (!nft_is_base_chain(trans->ctx.chain))
8510                 return;
8511
8512         nft_chain_stats_replace(trans);
8513
8514         basechain = nft_base_chain(trans->ctx.chain);
8515
8516         switch (nft_trans_chain_policy(trans)) {
8517         case NF_DROP:
8518         case NF_ACCEPT:
8519                 basechain->policy = nft_trans_chain_policy(trans);
8520                 break;
8521         }
8522 }
8523
8524 static void nft_obj_commit_update(struct nft_trans *trans)
8525 {
8526         struct nft_object *newobj;
8527         struct nft_object *obj;
8528
8529         obj = nft_trans_obj(trans);
8530         newobj = nft_trans_obj_newobj(trans);
8531
8532         if (obj->ops->update)
8533                 obj->ops->update(obj, newobj);
8534
8535         nft_obj_destroy(&trans->ctx, newobj);
8536 }
8537
8538 static void nft_commit_release(struct nft_trans *trans)
8539 {
8540         switch (trans->msg_type) {
8541         case NFT_MSG_DELTABLE:
8542                 nf_tables_table_destroy(&trans->ctx);
8543                 break;
8544         case NFT_MSG_NEWCHAIN:
8545                 free_percpu(nft_trans_chain_stats(trans));
8546                 kfree(nft_trans_chain_name(trans));
8547                 break;
8548         case NFT_MSG_DELCHAIN:
8549                 nf_tables_chain_destroy(&trans->ctx);
8550                 break;
8551         case NFT_MSG_DELRULE:
8552                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
8553                 break;
8554         case NFT_MSG_DELSET:
8555                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
8556                 break;
8557         case NFT_MSG_DELSETELEM:
8558                 nf_tables_set_elem_destroy(&trans->ctx,
8559                                            nft_trans_elem_set(trans),
8560                                            nft_trans_elem(trans).priv);
8561                 break;
8562         case NFT_MSG_DELOBJ:
8563                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
8564                 break;
8565         case NFT_MSG_DELFLOWTABLE:
8566                 if (nft_trans_flowtable_update(trans))
8567                         nft_flowtable_hooks_destroy(&nft_trans_flowtable_hooks(trans));
8568                 else
8569                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
8570                 break;
8571         }
8572
8573         if (trans->put_net)
8574                 put_net(trans->ctx.net);
8575
8576         kfree(trans);
8577 }
8578
8579 static void nf_tables_trans_destroy_work(struct work_struct *w)
8580 {
8581         struct nft_trans *trans, *next;
8582         LIST_HEAD(head);
8583
8584         spin_lock(&nf_tables_destroy_list_lock);
8585         list_splice_init(&nf_tables_destroy_list, &head);
8586         spin_unlock(&nf_tables_destroy_list_lock);
8587
8588         if (list_empty(&head))
8589                 return;
8590
8591         synchronize_rcu();
8592
8593         list_for_each_entry_safe(trans, next, &head, list) {
8594                 list_del(&trans->list);
8595                 nft_commit_release(trans);
8596         }
8597 }
8598
8599 void nf_tables_trans_destroy_flush_work(void)
8600 {
8601         flush_work(&trans_destroy_work);
8602 }
8603 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
8604
8605 static bool nft_expr_reduce(struct nft_regs_track *track,
8606                             const struct nft_expr *expr)
8607 {
8608         return false;
8609 }
8610
8611 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
8612 {
8613         const struct nft_expr *expr, *last;
8614         struct nft_regs_track track = {};
8615         unsigned int size, data_size;
8616         void *data, *data_boundary;
8617         struct nft_rule_dp *prule;
8618         struct nft_rule *rule;
8619
8620         /* already handled or inactive chain? */
8621         if (chain->blob_next || !nft_is_active_next(net, chain))
8622                 return 0;
8623
8624         data_size = 0;
8625         list_for_each_entry(rule, &chain->rules, list) {
8626                 if (nft_is_active_next(net, rule)) {
8627                         data_size += sizeof(*prule) + rule->dlen;
8628                         if (data_size > INT_MAX)
8629                                 return -ENOMEM;
8630                 }
8631         }
8632         data_size += offsetof(struct nft_rule_dp, data);        /* last rule */
8633
8634         chain->blob_next = nf_tables_chain_alloc_rules(data_size);
8635         if (!chain->blob_next)
8636                 return -ENOMEM;
8637
8638         data = (void *)chain->blob_next->data;
8639         data_boundary = data + data_size;
8640         size = 0;
8641
8642         list_for_each_entry(rule, &chain->rules, list) {
8643                 if (!nft_is_active_next(net, rule))
8644                         continue;
8645
8646                 prule = (struct nft_rule_dp *)data;
8647                 data += offsetof(struct nft_rule_dp, data);
8648                 if (WARN_ON_ONCE(data > data_boundary))
8649                         return -ENOMEM;
8650
8651                 size = 0;
8652                 track.last = nft_expr_last(rule);
8653                 nft_rule_for_each_expr(expr, last, rule) {
8654                         track.cur = expr;
8655
8656                         if (nft_expr_reduce(&track, expr)) {
8657                                 expr = track.cur;
8658                                 continue;
8659                         }
8660
8661                         if (WARN_ON_ONCE(data + expr->ops->size > data_boundary))
8662                                 return -ENOMEM;
8663
8664                         memcpy(data + size, expr, expr->ops->size);
8665                         size += expr->ops->size;
8666                 }
8667                 if (WARN_ON_ONCE(size >= 1 << 12))
8668                         return -ENOMEM;
8669
8670                 prule->handle = rule->handle;
8671                 prule->dlen = size;
8672                 prule->is_last = 0;
8673
8674                 data += size;
8675                 size = 0;
8676                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
8677         }
8678
8679         prule = (struct nft_rule_dp *)data;
8680         data += offsetof(struct nft_rule_dp, data);
8681         if (WARN_ON_ONCE(data > data_boundary))
8682                 return -ENOMEM;
8683
8684         nft_last_rule(chain->blob_next, prule);
8685
8686         return 0;
8687 }
8688
8689 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
8690 {
8691         struct nftables_pernet *nft_net = nft_pernet(net);
8692         struct nft_trans *trans, *next;
8693
8694         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
8695                 struct nft_chain *chain = trans->ctx.chain;
8696
8697                 if (trans->msg_type == NFT_MSG_NEWRULE ||
8698                     trans->msg_type == NFT_MSG_DELRULE) {
8699                         kvfree(chain->blob_next);
8700                         chain->blob_next = NULL;
8701                 }
8702         }
8703 }
8704
8705 static void __nf_tables_commit_chain_free_rules_old(struct rcu_head *h)
8706 {
8707         struct nft_rules_old *o = container_of(h, struct nft_rules_old, h);
8708
8709         kvfree(o->blob);
8710 }
8711
8712 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
8713 {
8714         struct nft_rules_old *old;
8715
8716         /* rcu_head is after end marker */
8717         old = (void *)blob + sizeof(*blob) + blob->size;
8718         old->blob = blob;
8719
8720         call_rcu(&old->h, __nf_tables_commit_chain_free_rules_old);
8721 }
8722
8723 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
8724 {
8725         struct nft_rule_blob *g0, *g1;
8726         bool next_genbit;
8727
8728         next_genbit = nft_gencursor_next(net);
8729
8730         g0 = rcu_dereference_protected(chain->blob_gen_0,
8731                                        lockdep_commit_lock_is_held(net));
8732         g1 = rcu_dereference_protected(chain->blob_gen_1,
8733                                        lockdep_commit_lock_is_held(net));
8734
8735         /* No changes to this chain? */
8736         if (chain->blob_next == NULL) {
8737                 /* chain had no change in last or next generation */
8738                 if (g0 == g1)
8739                         return;
8740                 /*
8741                  * chain had no change in this generation; make sure next
8742                  * one uses same rules as current generation.
8743                  */
8744                 if (next_genbit) {
8745                         rcu_assign_pointer(chain->blob_gen_1, g0);
8746                         nf_tables_commit_chain_free_rules_old(g1);
8747                 } else {
8748                         rcu_assign_pointer(chain->blob_gen_0, g1);
8749                         nf_tables_commit_chain_free_rules_old(g0);
8750                 }
8751
8752                 return;
8753         }
8754
8755         if (next_genbit)
8756                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
8757         else
8758                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
8759
8760         chain->blob_next = NULL;
8761
8762         if (g0 == g1)
8763                 return;
8764
8765         if (next_genbit)
8766                 nf_tables_commit_chain_free_rules_old(g1);
8767         else
8768                 nf_tables_commit_chain_free_rules_old(g0);
8769 }
8770
8771 static void nft_obj_del(struct nft_object *obj)
8772 {
8773         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
8774         list_del_rcu(&obj->list);
8775 }
8776
8777 void nft_chain_del(struct nft_chain *chain)
8778 {
8779         struct nft_table *table = chain->table;
8780
8781         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
8782                                      nft_chain_ht_params));
8783         list_del_rcu(&chain->list);
8784 }
8785
8786 static void nf_tables_module_autoload_cleanup(struct net *net)
8787 {
8788         struct nftables_pernet *nft_net = nft_pernet(net);
8789         struct nft_module_request *req, *next;
8790
8791         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
8792         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
8793                 WARN_ON_ONCE(!req->done);
8794                 list_del(&req->list);
8795                 kfree(req);
8796         }
8797 }
8798
8799 static void nf_tables_commit_release(struct net *net)
8800 {
8801         struct nftables_pernet *nft_net = nft_pernet(net);
8802         struct nft_trans *trans;
8803
8804         /* all side effects have to be made visible.
8805          * For example, if a chain named 'foo' has been deleted, a
8806          * new transaction must not find it anymore.
8807          *
8808          * Memory reclaim happens asynchronously from work queue
8809          * to prevent expensive synchronize_rcu() in commit phase.
8810          */
8811         if (list_empty(&nft_net->commit_list)) {
8812                 nf_tables_module_autoload_cleanup(net);
8813                 mutex_unlock(&nft_net->commit_mutex);
8814                 return;
8815         }
8816
8817         trans = list_last_entry(&nft_net->commit_list,
8818                                 struct nft_trans, list);
8819         get_net(trans->ctx.net);
8820         WARN_ON_ONCE(trans->put_net);
8821
8822         trans->put_net = true;
8823         spin_lock(&nf_tables_destroy_list_lock);
8824         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
8825         spin_unlock(&nf_tables_destroy_list_lock);
8826
8827         nf_tables_module_autoload_cleanup(net);
8828         schedule_work(&trans_destroy_work);
8829
8830         mutex_unlock(&nft_net->commit_mutex);
8831 }
8832
8833 static void nft_commit_notify(struct net *net, u32 portid)
8834 {
8835         struct nftables_pernet *nft_net = nft_pernet(net);
8836         struct sk_buff *batch_skb = NULL, *nskb, *skb;
8837         unsigned char *data;
8838         int len;
8839
8840         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
8841                 if (!batch_skb) {
8842 new_batch:
8843                         batch_skb = skb;
8844                         len = NLMSG_GOODSIZE - skb->len;
8845                         list_del(&skb->list);
8846                         continue;
8847                 }
8848                 len -= skb->len;
8849                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
8850                         data = skb_put(batch_skb, skb->len);
8851                         memcpy(data, skb->data, skb->len);
8852                         list_del(&skb->list);
8853                         kfree_skb(skb);
8854                         continue;
8855                 }
8856                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
8857                                NFT_CB(batch_skb).report, GFP_KERNEL);
8858                 goto new_batch;
8859         }
8860
8861         if (batch_skb) {
8862                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
8863                                NFT_CB(batch_skb).report, GFP_KERNEL);
8864         }
8865
8866         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
8867 }
8868
8869 static int nf_tables_commit_audit_alloc(struct list_head *adl,
8870                                         struct nft_table *table)
8871 {
8872         struct nft_audit_data *adp;
8873
8874         list_for_each_entry(adp, adl, list) {
8875                 if (adp->table == table)
8876                         return 0;
8877         }
8878         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
8879         if (!adp)
8880                 return -ENOMEM;
8881         adp->table = table;
8882         list_add(&adp->list, adl);
8883         return 0;
8884 }
8885
8886 static void nf_tables_commit_audit_free(struct list_head *adl)
8887 {
8888         struct nft_audit_data *adp, *adn;
8889
8890         list_for_each_entry_safe(adp, adn, adl, list) {
8891                 list_del(&adp->list);
8892                 kfree(adp);
8893         }
8894 }
8895
8896 static void nf_tables_commit_audit_collect(struct list_head *adl,
8897                                            struct nft_table *table, u32 op)
8898 {
8899         struct nft_audit_data *adp;
8900
8901         list_for_each_entry(adp, adl, list) {
8902                 if (adp->table == table)
8903                         goto found;
8904         }
8905         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
8906         return;
8907 found:
8908         adp->entries++;
8909         if (!adp->op || adp->op > op)
8910                 adp->op = op;
8911 }
8912
8913 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
8914
8915 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
8916 {
8917         struct nft_audit_data *adp, *adn;
8918         char aubuf[AUNFTABLENAMELEN];
8919
8920         list_for_each_entry_safe(adp, adn, adl, list) {
8921                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
8922                          generation);
8923                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
8924                                 nft2audit_op[adp->op], GFP_KERNEL);
8925                 list_del(&adp->list);
8926                 kfree(adp);
8927         }
8928 }
8929
8930 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
8931 {
8932         struct nftables_pernet *nft_net = nft_pernet(net);
8933         struct nft_trans *trans, *next;
8934         struct nft_trans_elem *te;
8935         struct nft_chain *chain;
8936         struct nft_table *table;
8937         unsigned int base_seq;
8938         LIST_HEAD(adl);
8939         int err;
8940
8941         if (list_empty(&nft_net->commit_list)) {
8942                 mutex_unlock(&nft_net->commit_mutex);
8943                 return 0;
8944         }
8945
8946         /* 0. Validate ruleset, otherwise roll back for error reporting. */
8947         if (nf_tables_validate(net) < 0)
8948                 return -EAGAIN;
8949
8950         err = nft_flow_rule_offload_commit(net);
8951         if (err < 0)
8952                 return err;
8953
8954         /* 1.  Allocate space for next generation rules_gen_X[] */
8955         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
8956                 int ret;
8957
8958                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
8959                 if (ret) {
8960                         nf_tables_commit_chain_prepare_cancel(net);
8961                         nf_tables_commit_audit_free(&adl);
8962                         return ret;
8963                 }
8964                 if (trans->msg_type == NFT_MSG_NEWRULE ||
8965                     trans->msg_type == NFT_MSG_DELRULE) {
8966                         chain = trans->ctx.chain;
8967
8968                         ret = nf_tables_commit_chain_prepare(net, chain);
8969                         if (ret < 0) {
8970                                 nf_tables_commit_chain_prepare_cancel(net);
8971                                 nf_tables_commit_audit_free(&adl);
8972                                 return ret;
8973                         }
8974                 }
8975         }
8976
8977         /* step 2.  Make rules_gen_X visible to packet path */
8978         list_for_each_entry(table, &nft_net->tables, list) {
8979                 list_for_each_entry(chain, &table->chains, list)
8980                         nf_tables_commit_chain(net, chain);
8981         }
8982
8983         /*
8984          * Bump generation counter, invalidate any dump in progress.
8985          * Cannot fail after this point.
8986          */
8987         base_seq = READ_ONCE(nft_net->base_seq);
8988         while (++base_seq == 0)
8989                 ;
8990
8991         WRITE_ONCE(nft_net->base_seq, base_seq);
8992
8993         /* step 3. Start new generation, rules_gen_X now in use. */
8994         net->nft.gencursor = nft_gencursor_next(net);
8995
8996         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
8997                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
8998                                                trans->msg_type);
8999                 switch (trans->msg_type) {
9000                 case NFT_MSG_NEWTABLE:
9001                         if (nft_trans_table_update(trans)) {
9002                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9003                                         nft_trans_destroy(trans);
9004                                         break;
9005                                 }
9006                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
9007                                         nf_tables_table_disable(net, trans->ctx.table);
9008
9009                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9010                         } else {
9011                                 nft_clear(net, trans->ctx.table);
9012                         }
9013                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
9014                         nft_trans_destroy(trans);
9015                         break;
9016                 case NFT_MSG_DELTABLE:
9017                         list_del_rcu(&trans->ctx.table->list);
9018                         nf_tables_table_notify(&trans->ctx, NFT_MSG_DELTABLE);
9019                         break;
9020                 case NFT_MSG_NEWCHAIN:
9021                         if (nft_trans_chain_update(trans)) {
9022                                 nft_chain_commit_update(trans);
9023                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN);
9024                                 /* trans destroyed after rcu grace period */
9025                         } else {
9026                                 nft_chain_commit_drop_policy(trans);
9027                                 nft_clear(net, trans->ctx.chain);
9028                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN);
9029                                 nft_trans_destroy(trans);
9030                         }
9031                         break;
9032                 case NFT_MSG_DELCHAIN:
9033                         nft_chain_del(trans->ctx.chain);
9034                         nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN);
9035                         nf_tables_unregister_hook(trans->ctx.net,
9036                                                   trans->ctx.table,
9037                                                   trans->ctx.chain);
9038                         break;
9039                 case NFT_MSG_NEWRULE:
9040                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9041                         nf_tables_rule_notify(&trans->ctx,
9042                                               nft_trans_rule(trans),
9043                                               NFT_MSG_NEWRULE);
9044                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9045                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9046
9047                         nft_trans_destroy(trans);
9048                         break;
9049                 case NFT_MSG_DELRULE:
9050                         list_del_rcu(&nft_trans_rule(trans)->list);
9051                         nf_tables_rule_notify(&trans->ctx,
9052                                               nft_trans_rule(trans),
9053                                               NFT_MSG_DELRULE);
9054                         nft_rule_expr_deactivate(&trans->ctx,
9055                                                  nft_trans_rule(trans),
9056                                                  NFT_TRANS_COMMIT);
9057
9058                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9059                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9060                         break;
9061                 case NFT_MSG_NEWSET:
9062                         if (nft_trans_set_update(trans)) {
9063                                 struct nft_set *set = nft_trans_set(trans);
9064
9065                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
9066                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
9067                         } else {
9068                                 nft_clear(net, nft_trans_set(trans));
9069                                 /* This avoids hitting -EBUSY when deleting the table
9070                                  * from the transaction.
9071                                  */
9072                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
9073                                     !list_empty(&nft_trans_set(trans)->bindings))
9074                                         trans->ctx.table->use--;
9075                         }
9076                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9077                                              NFT_MSG_NEWSET, GFP_KERNEL);
9078                         nft_trans_destroy(trans);
9079                         break;
9080                 case NFT_MSG_DELSET:
9081                         list_del_rcu(&nft_trans_set(trans)->list);
9082                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9083                                              NFT_MSG_DELSET, GFP_KERNEL);
9084                         break;
9085                 case NFT_MSG_NEWSETELEM:
9086                         te = (struct nft_trans_elem *)trans->data;
9087
9088                         nft_setelem_activate(net, te->set, &te->elem);
9089                         nf_tables_setelem_notify(&trans->ctx, te->set,
9090                                                  &te->elem,
9091                                                  NFT_MSG_NEWSETELEM);
9092                         nft_trans_destroy(trans);
9093                         break;
9094                 case NFT_MSG_DELSETELEM:
9095                         te = (struct nft_trans_elem *)trans->data;
9096
9097                         nf_tables_setelem_notify(&trans->ctx, te->set,
9098                                                  &te->elem,
9099                                                  NFT_MSG_DELSETELEM);
9100                         nft_setelem_remove(net, te->set, &te->elem);
9101                         if (!nft_setelem_is_catchall(te->set, &te->elem)) {
9102                                 atomic_dec(&te->set->nelems);
9103                                 te->set->ndeact--;
9104                         }
9105                         break;
9106                 case NFT_MSG_NEWOBJ:
9107                         if (nft_trans_obj_update(trans)) {
9108                                 nft_obj_commit_update(trans);
9109                                 nf_tables_obj_notify(&trans->ctx,
9110                                                      nft_trans_obj(trans),
9111                                                      NFT_MSG_NEWOBJ);
9112                         } else {
9113                                 nft_clear(net, nft_trans_obj(trans));
9114                                 nf_tables_obj_notify(&trans->ctx,
9115                                                      nft_trans_obj(trans),
9116                                                      NFT_MSG_NEWOBJ);
9117                                 nft_trans_destroy(trans);
9118                         }
9119                         break;
9120                 case NFT_MSG_DELOBJ:
9121                         nft_obj_del(nft_trans_obj(trans));
9122                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
9123                                              NFT_MSG_DELOBJ);
9124                         break;
9125                 case NFT_MSG_NEWFLOWTABLE:
9126                         if (nft_trans_flowtable_update(trans)) {
9127                                 nft_trans_flowtable(trans)->data.flags =
9128                                         nft_trans_flowtable_flags(trans);
9129                                 nf_tables_flowtable_notify(&trans->ctx,
9130                                                            nft_trans_flowtable(trans),
9131                                                            &nft_trans_flowtable_hooks(trans),
9132                                                            NFT_MSG_NEWFLOWTABLE);
9133                                 list_splice(&nft_trans_flowtable_hooks(trans),
9134                                             &nft_trans_flowtable(trans)->hook_list);
9135                         } else {
9136                                 nft_clear(net, nft_trans_flowtable(trans));
9137                                 nf_tables_flowtable_notify(&trans->ctx,
9138                                                            nft_trans_flowtable(trans),
9139                                                            &nft_trans_flowtable(trans)->hook_list,
9140                                                            NFT_MSG_NEWFLOWTABLE);
9141                         }
9142                         nft_trans_destroy(trans);
9143                         break;
9144                 case NFT_MSG_DELFLOWTABLE:
9145                         if (nft_trans_flowtable_update(trans)) {
9146                                 nf_tables_flowtable_notify(&trans->ctx,
9147                                                            nft_trans_flowtable(trans),
9148                                                            &nft_trans_flowtable_hooks(trans),
9149                                                            NFT_MSG_DELFLOWTABLE);
9150                                 nft_unregister_flowtable_net_hooks(net,
9151                                                                    &nft_trans_flowtable_hooks(trans));
9152                         } else {
9153                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
9154                                 nf_tables_flowtable_notify(&trans->ctx,
9155                                                            nft_trans_flowtable(trans),
9156                                                            &nft_trans_flowtable(trans)->hook_list,
9157                                                            NFT_MSG_DELFLOWTABLE);
9158                                 nft_unregister_flowtable_net_hooks(net,
9159                                                 &nft_trans_flowtable(trans)->hook_list);
9160                         }
9161                         break;
9162                 }
9163         }
9164
9165         nft_commit_notify(net, NETLINK_CB(skb).portid);
9166         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
9167         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
9168         nf_tables_commit_release(net);
9169
9170         return 0;
9171 }
9172
9173 static void nf_tables_module_autoload(struct net *net)
9174 {
9175         struct nftables_pernet *nft_net = nft_pernet(net);
9176         struct nft_module_request *req, *next;
9177         LIST_HEAD(module_list);
9178
9179         list_splice_init(&nft_net->module_list, &module_list);
9180         mutex_unlock(&nft_net->commit_mutex);
9181         list_for_each_entry_safe(req, next, &module_list, list) {
9182                 request_module("%s", req->module);
9183                 req->done = true;
9184         }
9185         mutex_lock(&nft_net->commit_mutex);
9186         list_splice(&module_list, &nft_net->module_list);
9187 }
9188
9189 static void nf_tables_abort_release(struct nft_trans *trans)
9190 {
9191         switch (trans->msg_type) {
9192         case NFT_MSG_NEWTABLE:
9193                 nf_tables_table_destroy(&trans->ctx);
9194                 break;
9195         case NFT_MSG_NEWCHAIN:
9196                 nf_tables_chain_destroy(&trans->ctx);
9197                 break;
9198         case NFT_MSG_NEWRULE:
9199                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9200                 break;
9201         case NFT_MSG_NEWSET:
9202                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9203                 break;
9204         case NFT_MSG_NEWSETELEM:
9205                 nft_set_elem_destroy(nft_trans_elem_set(trans),
9206                                      nft_trans_elem(trans).priv, true);
9207                 break;
9208         case NFT_MSG_NEWOBJ:
9209                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9210                 break;
9211         case NFT_MSG_NEWFLOWTABLE:
9212                 if (nft_trans_flowtable_update(trans))
9213                         nft_flowtable_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9214                 else
9215                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9216                 break;
9217         }
9218         kfree(trans);
9219 }
9220
9221 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
9222 {
9223         struct nftables_pernet *nft_net = nft_pernet(net);
9224         struct nft_trans *trans, *next;
9225         struct nft_trans_elem *te;
9226
9227         if (action == NFNL_ABORT_VALIDATE &&
9228             nf_tables_validate(net) < 0)
9229                 return -EAGAIN;
9230
9231         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
9232                                          list) {
9233                 switch (trans->msg_type) {
9234                 case NFT_MSG_NEWTABLE:
9235                         if (nft_trans_table_update(trans)) {
9236                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
9237                                         nft_trans_destroy(trans);
9238                                         break;
9239                                 }
9240                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
9241                                         nf_tables_table_disable(net, trans->ctx.table);
9242                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
9243                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
9244                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
9245                                 }
9246                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
9247                                 nft_trans_destroy(trans);
9248                         } else {
9249                                 list_del_rcu(&trans->ctx.table->list);
9250                         }
9251                         break;
9252                 case NFT_MSG_DELTABLE:
9253                         nft_clear(trans->ctx.net, trans->ctx.table);
9254                         nft_trans_destroy(trans);
9255                         break;
9256                 case NFT_MSG_NEWCHAIN:
9257                         if (nft_trans_chain_update(trans)) {
9258                                 free_percpu(nft_trans_chain_stats(trans));
9259                                 kfree(nft_trans_chain_name(trans));
9260                                 nft_trans_destroy(trans);
9261                         } else {
9262                                 if (nft_chain_is_bound(trans->ctx.chain)) {
9263                                         nft_trans_destroy(trans);
9264                                         break;
9265                                 }
9266                                 trans->ctx.table->use--;
9267                                 nft_chain_del(trans->ctx.chain);
9268                                 nf_tables_unregister_hook(trans->ctx.net,
9269                                                           trans->ctx.table,
9270                                                           trans->ctx.chain);
9271                         }
9272                         break;
9273                 case NFT_MSG_DELCHAIN:
9274                         trans->ctx.table->use++;
9275                         nft_clear(trans->ctx.net, trans->ctx.chain);
9276                         nft_trans_destroy(trans);
9277                         break;
9278                 case NFT_MSG_NEWRULE:
9279                         trans->ctx.chain->use--;
9280                         list_del_rcu(&nft_trans_rule(trans)->list);
9281                         nft_rule_expr_deactivate(&trans->ctx,
9282                                                  nft_trans_rule(trans),
9283                                                  NFT_TRANS_ABORT);
9284                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9285                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9286                         break;
9287                 case NFT_MSG_DELRULE:
9288                         trans->ctx.chain->use++;
9289                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
9290                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
9291                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
9292                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
9293
9294                         nft_trans_destroy(trans);
9295                         break;
9296                 case NFT_MSG_NEWSET:
9297                         if (nft_trans_set_update(trans)) {
9298                                 nft_trans_destroy(trans);
9299                                 break;
9300                         }
9301                         trans->ctx.table->use--;
9302                         if (nft_trans_set_bound(trans)) {
9303                                 nft_trans_destroy(trans);
9304                                 break;
9305                         }
9306                         list_del_rcu(&nft_trans_set(trans)->list);
9307                         break;
9308                 case NFT_MSG_DELSET:
9309                         trans->ctx.table->use++;
9310                         nft_clear(trans->ctx.net, nft_trans_set(trans));
9311                         nft_trans_destroy(trans);
9312                         break;
9313                 case NFT_MSG_NEWSETELEM:
9314                         if (nft_trans_elem_set_bound(trans)) {
9315                                 nft_trans_destroy(trans);
9316                                 break;
9317                         }
9318                         te = (struct nft_trans_elem *)trans->data;
9319                         nft_setelem_remove(net, te->set, &te->elem);
9320                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9321                                 atomic_dec(&te->set->nelems);
9322                         break;
9323                 case NFT_MSG_DELSETELEM:
9324                         te = (struct nft_trans_elem *)trans->data;
9325
9326                         nft_setelem_data_activate(net, te->set, &te->elem);
9327                         nft_setelem_activate(net, te->set, &te->elem);
9328                         if (!nft_setelem_is_catchall(te->set, &te->elem))
9329                                 te->set->ndeact--;
9330
9331                         nft_trans_destroy(trans);
9332                         break;
9333                 case NFT_MSG_NEWOBJ:
9334                         if (nft_trans_obj_update(trans)) {
9335                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
9336                                 nft_trans_destroy(trans);
9337                         } else {
9338                                 trans->ctx.table->use--;
9339                                 nft_obj_del(nft_trans_obj(trans));
9340                         }
9341                         break;
9342                 case NFT_MSG_DELOBJ:
9343                         trans->ctx.table->use++;
9344                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
9345                         nft_trans_destroy(trans);
9346                         break;
9347                 case NFT_MSG_NEWFLOWTABLE:
9348                         if (nft_trans_flowtable_update(trans)) {
9349                                 nft_unregister_flowtable_net_hooks(net,
9350                                                 &nft_trans_flowtable_hooks(trans));
9351                         } else {
9352                                 trans->ctx.table->use--;
9353                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
9354                                 nft_unregister_flowtable_net_hooks(net,
9355                                                 &nft_trans_flowtable(trans)->hook_list);
9356                         }
9357                         break;
9358                 case NFT_MSG_DELFLOWTABLE:
9359                         if (nft_trans_flowtable_update(trans)) {
9360                                 list_splice(&nft_trans_flowtable_hooks(trans),
9361                                             &nft_trans_flowtable(trans)->hook_list);
9362                         } else {
9363                                 trans->ctx.table->use++;
9364                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
9365                         }
9366                         nft_trans_destroy(trans);
9367                         break;
9368                 }
9369         }
9370
9371         synchronize_rcu();
9372
9373         list_for_each_entry_safe_reverse(trans, next,
9374                                          &nft_net->commit_list, list) {
9375                 list_del(&trans->list);
9376                 nf_tables_abort_release(trans);
9377         }
9378
9379         if (action == NFNL_ABORT_AUTOLOAD)
9380                 nf_tables_module_autoload(net);
9381         else
9382                 nf_tables_module_autoload_cleanup(net);
9383
9384         return 0;
9385 }
9386
9387 static void nf_tables_cleanup(struct net *net)
9388 {
9389         nft_validate_state_update(net, NFT_VALIDATE_SKIP);
9390 }
9391
9392 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
9393                            enum nfnl_abort_action action)
9394 {
9395         struct nftables_pernet *nft_net = nft_pernet(net);
9396         int ret = __nf_tables_abort(net, action);
9397
9398         mutex_unlock(&nft_net->commit_mutex);
9399
9400         return ret;
9401 }
9402
9403 static bool nf_tables_valid_genid(struct net *net, u32 genid)
9404 {
9405         struct nftables_pernet *nft_net = nft_pernet(net);
9406         bool genid_ok;
9407
9408         mutex_lock(&nft_net->commit_mutex);
9409
9410         genid_ok = genid == 0 || nft_net->base_seq == genid;
9411         if (!genid_ok)
9412                 mutex_unlock(&nft_net->commit_mutex);
9413
9414         /* else, commit mutex has to be released by commit or abort function */
9415         return genid_ok;
9416 }
9417
9418 static const struct nfnetlink_subsystem nf_tables_subsys = {
9419         .name           = "nf_tables",
9420         .subsys_id      = NFNL_SUBSYS_NFTABLES,
9421         .cb_count       = NFT_MSG_MAX,
9422         .cb             = nf_tables_cb,
9423         .commit         = nf_tables_commit,
9424         .abort          = nf_tables_abort,
9425         .cleanup        = nf_tables_cleanup,
9426         .valid_genid    = nf_tables_valid_genid,
9427         .owner          = THIS_MODULE,
9428 };
9429
9430 int nft_chain_validate_dependency(const struct nft_chain *chain,
9431                                   enum nft_chain_types type)
9432 {
9433         const struct nft_base_chain *basechain;
9434
9435         if (nft_is_base_chain(chain)) {
9436                 basechain = nft_base_chain(chain);
9437                 if (basechain->type->type != type)
9438                         return -EOPNOTSUPP;
9439         }
9440         return 0;
9441 }
9442 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
9443
9444 int nft_chain_validate_hooks(const struct nft_chain *chain,
9445                              unsigned int hook_flags)
9446 {
9447         struct nft_base_chain *basechain;
9448
9449         if (nft_is_base_chain(chain)) {
9450                 basechain = nft_base_chain(chain);
9451
9452                 if ((1 << basechain->ops.hooknum) & hook_flags)
9453                         return 0;
9454
9455                 return -EOPNOTSUPP;
9456         }
9457
9458         return 0;
9459 }
9460 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
9461
9462 /*
9463  * Loop detection - walk through the ruleset beginning at the destination chain
9464  * of a new jump until either the source chain is reached (loop) or all
9465  * reachable chains have been traversed.
9466  *
9467  * The loop check is performed whenever a new jump verdict is added to an
9468  * expression or verdict map or a verdict map is bound to a new chain.
9469  */
9470
9471 static int nf_tables_check_loops(const struct nft_ctx *ctx,
9472                                  const struct nft_chain *chain);
9473
9474 static int nft_check_loops(const struct nft_ctx *ctx,
9475                            const struct nft_set_ext *ext)
9476 {
9477         const struct nft_data *data;
9478         int ret;
9479
9480         data = nft_set_ext_data(ext);
9481         switch (data->verdict.code) {
9482         case NFT_JUMP:
9483         case NFT_GOTO:
9484                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
9485                 break;
9486         default:
9487                 ret = 0;
9488                 break;
9489         }
9490
9491         return ret;
9492 }
9493
9494 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
9495                                         struct nft_set *set,
9496                                         const struct nft_set_iter *iter,
9497                                         struct nft_set_elem *elem)
9498 {
9499         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
9500
9501         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
9502             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
9503                 return 0;
9504
9505         return nft_check_loops(ctx, ext);
9506 }
9507
9508 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
9509                                   struct nft_set *set)
9510 {
9511         u8 genmask = nft_genmask_next(ctx->net);
9512         struct nft_set_elem_catchall *catchall;
9513         struct nft_set_ext *ext;
9514         int ret = 0;
9515
9516         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9517                 ext = nft_set_elem_ext(set, catchall->elem);
9518                 if (!nft_set_elem_active(ext, genmask))
9519                         continue;
9520
9521                 ret = nft_check_loops(ctx, ext);
9522                 if (ret < 0)
9523                         return ret;
9524         }
9525
9526         return ret;
9527 }
9528
9529 static int nf_tables_check_loops(const struct nft_ctx *ctx,
9530                                  const struct nft_chain *chain)
9531 {
9532         const struct nft_rule *rule;
9533         const struct nft_expr *expr, *last;
9534         struct nft_set *set;
9535         struct nft_set_binding *binding;
9536         struct nft_set_iter iter;
9537
9538         if (ctx->chain == chain)
9539                 return -ELOOP;
9540
9541         list_for_each_entry(rule, &chain->rules, list) {
9542                 nft_rule_for_each_expr(expr, last, rule) {
9543                         struct nft_immediate_expr *priv;
9544                         const struct nft_data *data;
9545                         int err;
9546
9547                         if (strcmp(expr->ops->type->name, "immediate"))
9548                                 continue;
9549
9550                         priv = nft_expr_priv(expr);
9551                         if (priv->dreg != NFT_REG_VERDICT)
9552                                 continue;
9553
9554                         data = &priv->data;
9555                         switch (data->verdict.code) {
9556                         case NFT_JUMP:
9557                         case NFT_GOTO:
9558                                 err = nf_tables_check_loops(ctx,
9559                                                         data->verdict.chain);
9560                                 if (err < 0)
9561                                         return err;
9562                                 break;
9563                         default:
9564                                 break;
9565                         }
9566                 }
9567         }
9568
9569         list_for_each_entry(set, &ctx->table->sets, list) {
9570                 if (!nft_is_active_next(ctx->net, set))
9571                         continue;
9572                 if (!(set->flags & NFT_SET_MAP) ||
9573                     set->dtype != NFT_DATA_VERDICT)
9574                         continue;
9575
9576                 list_for_each_entry(binding, &set->bindings, list) {
9577                         if (!(binding->flags & NFT_SET_MAP) ||
9578                             binding->chain != chain)
9579                                 continue;
9580
9581                         iter.genmask    = nft_genmask_next(ctx->net);
9582                         iter.skip       = 0;
9583                         iter.count      = 0;
9584                         iter.err        = 0;
9585                         iter.fn         = nf_tables_loop_check_setelem;
9586
9587                         set->ops->walk(ctx, set, &iter);
9588                         if (!iter.err)
9589                                 iter.err = nft_set_catchall_loops(ctx, set);
9590
9591                         if (iter.err < 0)
9592                                 return iter.err;
9593                 }
9594         }
9595
9596         return 0;
9597 }
9598
9599 /**
9600  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
9601  *
9602  *      @attr: netlink attribute to fetch value from
9603  *      @max: maximum value to be stored in dest
9604  *      @dest: pointer to the variable
9605  *
9606  *      Parse, check and store a given u32 netlink attribute into variable.
9607  *      This function returns -ERANGE if the value goes over maximum value.
9608  *      Otherwise a 0 is returned and the attribute value is stored in the
9609  *      destination variable.
9610  */
9611 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
9612 {
9613         u32 val;
9614
9615         val = ntohl(nla_get_be32(attr));
9616         if (val > max)
9617                 return -ERANGE;
9618
9619         *dest = val;
9620         return 0;
9621 }
9622 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
9623
9624 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
9625 {
9626         unsigned int reg;
9627
9628         reg = ntohl(nla_get_be32(attr));
9629         switch (reg) {
9630         case NFT_REG_VERDICT...NFT_REG_4:
9631                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
9632                 break;
9633         case NFT_REG32_00...NFT_REG32_15:
9634                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
9635                 break;
9636         default:
9637                 return -ERANGE;
9638         }
9639
9640         return 0;
9641 }
9642
9643 /**
9644  *      nft_dump_register - dump a register value to a netlink attribute
9645  *
9646  *      @skb: socket buffer
9647  *      @attr: attribute number
9648  *      @reg: register number
9649  *
9650  *      Construct a netlink attribute containing the register number. For
9651  *      compatibility reasons, register numbers being a multiple of 4 are
9652  *      translated to the corresponding 128 bit register numbers.
9653  */
9654 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
9655 {
9656         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
9657                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
9658         else
9659                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
9660
9661         return nla_put_be32(skb, attr, htonl(reg));
9662 }
9663 EXPORT_SYMBOL_GPL(nft_dump_register);
9664
9665 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
9666 {
9667         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
9668                 return -EINVAL;
9669         if (len == 0)
9670                 return -EINVAL;
9671         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
9672                 return -ERANGE;
9673
9674         return 0;
9675 }
9676
9677 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
9678 {
9679         u32 reg;
9680         int err;
9681
9682         err = nft_parse_register(attr, &reg);
9683         if (err < 0)
9684                 return err;
9685
9686         err = nft_validate_register_load(reg, len);
9687         if (err < 0)
9688                 return err;
9689
9690         *sreg = reg;
9691         return 0;
9692 }
9693 EXPORT_SYMBOL_GPL(nft_parse_register_load);
9694
9695 static int nft_validate_register_store(const struct nft_ctx *ctx,
9696                                        enum nft_registers reg,
9697                                        const struct nft_data *data,
9698                                        enum nft_data_types type,
9699                                        unsigned int len)
9700 {
9701         int err;
9702
9703         switch (reg) {
9704         case NFT_REG_VERDICT:
9705                 if (type != NFT_DATA_VERDICT)
9706                         return -EINVAL;
9707
9708                 if (data != NULL &&
9709                     (data->verdict.code == NFT_GOTO ||
9710                      data->verdict.code == NFT_JUMP)) {
9711                         err = nf_tables_check_loops(ctx, data->verdict.chain);
9712                         if (err < 0)
9713                                 return err;
9714                 }
9715
9716                 return 0;
9717         default:
9718                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
9719                         return -EINVAL;
9720                 if (len == 0)
9721                         return -EINVAL;
9722                 if (reg * NFT_REG32_SIZE + len >
9723                     sizeof_field(struct nft_regs, data))
9724                         return -ERANGE;
9725
9726                 if (data != NULL && type != NFT_DATA_VALUE)
9727                         return -EINVAL;
9728                 return 0;
9729         }
9730 }
9731
9732 int nft_parse_register_store(const struct nft_ctx *ctx,
9733                              const struct nlattr *attr, u8 *dreg,
9734                              const struct nft_data *data,
9735                              enum nft_data_types type, unsigned int len)
9736 {
9737         int err;
9738         u32 reg;
9739
9740         err = nft_parse_register(attr, &reg);
9741         if (err < 0)
9742                 return err;
9743
9744         err = nft_validate_register_store(ctx, reg, data, type, len);
9745         if (err < 0)
9746                 return err;
9747
9748         *dreg = reg;
9749         return 0;
9750 }
9751 EXPORT_SYMBOL_GPL(nft_parse_register_store);
9752
9753 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
9754         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
9755         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
9756                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
9757         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
9758 };
9759
9760 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
9761                             struct nft_data_desc *desc, const struct nlattr *nla)
9762 {
9763         u8 genmask = nft_genmask_next(ctx->net);
9764         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
9765         struct nft_chain *chain;
9766         int err;
9767
9768         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
9769                                           nft_verdict_policy, NULL);
9770         if (err < 0)
9771                 return err;
9772
9773         if (!tb[NFTA_VERDICT_CODE])
9774                 return -EINVAL;
9775         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
9776
9777         switch (data->verdict.code) {
9778         default:
9779                 switch (data->verdict.code & NF_VERDICT_MASK) {
9780                 case NF_ACCEPT:
9781                 case NF_DROP:
9782                 case NF_QUEUE:
9783                         break;
9784                 default:
9785                         return -EINVAL;
9786                 }
9787                 fallthrough;
9788         case NFT_CONTINUE:
9789         case NFT_BREAK:
9790         case NFT_RETURN:
9791                 break;
9792         case NFT_JUMP:
9793         case NFT_GOTO:
9794                 if (tb[NFTA_VERDICT_CHAIN]) {
9795                         chain = nft_chain_lookup(ctx->net, ctx->table,
9796                                                  tb[NFTA_VERDICT_CHAIN],
9797                                                  genmask);
9798                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
9799                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
9800                                                       tb[NFTA_VERDICT_CHAIN_ID]);
9801                         if (IS_ERR(chain))
9802                                 return PTR_ERR(chain);
9803                 } else {
9804                         return -EINVAL;
9805                 }
9806
9807                 if (IS_ERR(chain))
9808                         return PTR_ERR(chain);
9809                 if (nft_is_base_chain(chain))
9810                         return -EOPNOTSUPP;
9811                 if (nft_chain_is_bound(chain))
9812                         return -EINVAL;
9813                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
9814                     chain->flags & NFT_CHAIN_BINDING)
9815                         return -EINVAL;
9816
9817                 chain->use++;
9818                 data->verdict.chain = chain;
9819                 break;
9820         }
9821
9822         desc->len = sizeof(data->verdict);
9823
9824         return 0;
9825 }
9826
9827 static void nft_verdict_uninit(const struct nft_data *data)
9828 {
9829         struct nft_chain *chain;
9830         struct nft_rule *rule;
9831
9832         switch (data->verdict.code) {
9833         case NFT_JUMP:
9834         case NFT_GOTO:
9835                 chain = data->verdict.chain;
9836                 chain->use--;
9837
9838                 if (!nft_chain_is_bound(chain))
9839                         break;
9840
9841                 chain->table->use--;
9842                 list_for_each_entry(rule, &chain->rules, list)
9843                         chain->use--;
9844
9845                 nft_chain_del(chain);
9846                 break;
9847         }
9848 }
9849
9850 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
9851 {
9852         struct nlattr *nest;
9853
9854         nest = nla_nest_start_noflag(skb, type);
9855         if (!nest)
9856                 goto nla_put_failure;
9857
9858         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
9859                 goto nla_put_failure;
9860
9861         switch (v->code) {
9862         case NFT_JUMP:
9863         case NFT_GOTO:
9864                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
9865                                    v->chain->name))
9866                         goto nla_put_failure;
9867         }
9868         nla_nest_end(skb, nest);
9869         return 0;
9870
9871 nla_put_failure:
9872         return -1;
9873 }
9874
9875 static int nft_value_init(const struct nft_ctx *ctx,
9876                           struct nft_data *data, struct nft_data_desc *desc,
9877                           const struct nlattr *nla)
9878 {
9879         unsigned int len;
9880
9881         len = nla_len(nla);
9882         if (len == 0)
9883                 return -EINVAL;
9884         if (len > desc->size)
9885                 return -EOVERFLOW;
9886         if (desc->len) {
9887                 if (len != desc->len)
9888                         return -EINVAL;
9889         } else {
9890                 desc->len = len;
9891         }
9892
9893         nla_memcpy(data->data, nla, len);
9894
9895         return 0;
9896 }
9897
9898 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
9899                           unsigned int len)
9900 {
9901         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
9902 }
9903
9904 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
9905         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
9906         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
9907 };
9908
9909 /**
9910  *      nft_data_init - parse nf_tables data netlink attributes
9911  *
9912  *      @ctx: context of the expression using the data
9913  *      @data: destination struct nft_data
9914  *      @desc: data description
9915  *      @nla: netlink attribute containing data
9916  *
9917  *      Parse the netlink data attributes and initialize a struct nft_data.
9918  *      The type and length of data are returned in the data description.
9919  *
9920  *      The caller can indicate that it only wants to accept data of type
9921  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
9922  */
9923 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
9924                   struct nft_data_desc *desc, const struct nlattr *nla)
9925 {
9926         struct nlattr *tb[NFTA_DATA_MAX + 1];
9927         int err;
9928
9929         if (WARN_ON_ONCE(!desc->size))
9930                 return -EINVAL;
9931
9932         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
9933                                           nft_data_policy, NULL);
9934         if (err < 0)
9935                 return err;
9936
9937         if (tb[NFTA_DATA_VALUE]) {
9938                 if (desc->type != NFT_DATA_VALUE)
9939                         return -EINVAL;
9940
9941                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
9942         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
9943                 if (desc->type != NFT_DATA_VERDICT)
9944                         return -EINVAL;
9945
9946                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
9947         } else {
9948                 err = -EINVAL;
9949         }
9950
9951         return err;
9952 }
9953 EXPORT_SYMBOL_GPL(nft_data_init);
9954
9955 /**
9956  *      nft_data_release - release a nft_data item
9957  *
9958  *      @data: struct nft_data to release
9959  *      @type: type of data
9960  *
9961  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
9962  *      all others need to be released by calling this function.
9963  */
9964 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
9965 {
9966         if (type < NFT_DATA_VERDICT)
9967                 return;
9968         switch (type) {
9969         case NFT_DATA_VERDICT:
9970                 return nft_verdict_uninit(data);
9971         default:
9972                 WARN_ON(1);
9973         }
9974 }
9975 EXPORT_SYMBOL_GPL(nft_data_release);
9976
9977 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
9978                   enum nft_data_types type, unsigned int len)
9979 {
9980         struct nlattr *nest;
9981         int err;
9982
9983         nest = nla_nest_start_noflag(skb, attr);
9984         if (nest == NULL)
9985                 return -1;
9986
9987         switch (type) {
9988         case NFT_DATA_VALUE:
9989                 err = nft_value_dump(skb, data, len);
9990                 break;
9991         case NFT_DATA_VERDICT:
9992                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
9993                 break;
9994         default:
9995                 err = -EINVAL;
9996                 WARN_ON(1);
9997         }
9998
9999         nla_nest_end(skb, nest);
10000         return err;
10001 }
10002 EXPORT_SYMBOL_GPL(nft_data_dump);
10003
10004 int __nft_release_basechain(struct nft_ctx *ctx)
10005 {
10006         struct nft_rule *rule, *nr;
10007
10008         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
10009                 return 0;
10010
10011         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
10012         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
10013                 list_del(&rule->list);
10014                 ctx->chain->use--;
10015                 nf_tables_rule_release(ctx, rule);
10016         }
10017         nft_chain_del(ctx->chain);
10018         ctx->table->use--;
10019         nf_tables_chain_destroy(ctx);
10020
10021         return 0;
10022 }
10023 EXPORT_SYMBOL_GPL(__nft_release_basechain);
10024
10025 static void __nft_release_hook(struct net *net, struct nft_table *table)
10026 {
10027         struct nft_flowtable *flowtable;
10028         struct nft_chain *chain;
10029
10030         list_for_each_entry(chain, &table->chains, list)
10031                 __nf_tables_unregister_hook(net, table, chain, true);
10032         list_for_each_entry(flowtable, &table->flowtables, list)
10033                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
10034                                                      true);
10035 }
10036
10037 static void __nft_release_hooks(struct net *net)
10038 {
10039         struct nftables_pernet *nft_net = nft_pernet(net);
10040         struct nft_table *table;
10041
10042         list_for_each_entry(table, &nft_net->tables, list) {
10043                 if (nft_table_has_owner(table))
10044                         continue;
10045
10046                 __nft_release_hook(net, table);
10047         }
10048 }
10049
10050 static void __nft_release_table(struct net *net, struct nft_table *table)
10051 {
10052         struct nft_flowtable *flowtable, *nf;
10053         struct nft_chain *chain, *nc;
10054         struct nft_object *obj, *ne;
10055         struct nft_rule *rule, *nr;
10056         struct nft_set *set, *ns;
10057         struct nft_ctx ctx = {
10058                 .net    = net,
10059                 .family = NFPROTO_NETDEV,
10060         };
10061
10062         ctx.family = table->family;
10063         ctx.table = table;
10064         list_for_each_entry(chain, &table->chains, list) {
10065                 ctx.chain = chain;
10066                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
10067                         list_del(&rule->list);
10068                         chain->use--;
10069                         nf_tables_rule_release(&ctx, rule);
10070                 }
10071         }
10072         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
10073                 list_del(&flowtable->list);
10074                 table->use--;
10075                 nf_tables_flowtable_destroy(flowtable);
10076         }
10077         list_for_each_entry_safe(set, ns, &table->sets, list) {
10078                 list_del(&set->list);
10079                 table->use--;
10080                 nft_set_destroy(&ctx, set);
10081         }
10082         list_for_each_entry_safe(obj, ne, &table->objects, list) {
10083                 nft_obj_del(obj);
10084                 table->use--;
10085                 nft_obj_destroy(&ctx, obj);
10086         }
10087         list_for_each_entry_safe(chain, nc, &table->chains, list) {
10088                 ctx.chain = chain;
10089                 nft_chain_del(chain);
10090                 table->use--;
10091                 nf_tables_chain_destroy(&ctx);
10092         }
10093         nf_tables_table_destroy(&ctx);
10094 }
10095
10096 static void __nft_release_tables(struct net *net)
10097 {
10098         struct nftables_pernet *nft_net = nft_pernet(net);
10099         struct nft_table *table, *nt;
10100
10101         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
10102                 if (nft_table_has_owner(table))
10103                         continue;
10104
10105                 list_del(&table->list);
10106
10107                 __nft_release_table(net, table);
10108         }
10109 }
10110
10111 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
10112                             void *ptr)
10113 {
10114         struct nft_table *table, *to_delete[8];
10115         struct nftables_pernet *nft_net;
10116         struct netlink_notify *n = ptr;
10117         struct net *net = n->net;
10118         unsigned int deleted;
10119         bool restart = false;
10120
10121         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
10122                 return NOTIFY_DONE;
10123
10124         nft_net = nft_pernet(net);
10125         deleted = 0;
10126         mutex_lock(&nft_net->commit_mutex);
10127         if (!list_empty(&nf_tables_destroy_list))
10128                 rcu_barrier();
10129 again:
10130         list_for_each_entry(table, &nft_net->tables, list) {
10131                 if (nft_table_has_owner(table) &&
10132                     n->portid == table->nlpid) {
10133                         __nft_release_hook(net, table);
10134                         list_del_rcu(&table->list);
10135                         to_delete[deleted++] = table;
10136                         if (deleted >= ARRAY_SIZE(to_delete))
10137                                 break;
10138                 }
10139         }
10140         if (deleted) {
10141                 restart = deleted >= ARRAY_SIZE(to_delete);
10142                 synchronize_rcu();
10143                 while (deleted)
10144                         __nft_release_table(net, to_delete[--deleted]);
10145
10146                 if (restart)
10147                         goto again;
10148         }
10149         mutex_unlock(&nft_net->commit_mutex);
10150
10151         return NOTIFY_DONE;
10152 }
10153
10154 static struct notifier_block nft_nl_notifier = {
10155         .notifier_call  = nft_rcv_nl_event,
10156 };
10157
10158 static int __net_init nf_tables_init_net(struct net *net)
10159 {
10160         struct nftables_pernet *nft_net = nft_pernet(net);
10161
10162         INIT_LIST_HEAD(&nft_net->tables);
10163         INIT_LIST_HEAD(&nft_net->commit_list);
10164         INIT_LIST_HEAD(&nft_net->module_list);
10165         INIT_LIST_HEAD(&nft_net->notify_list);
10166         mutex_init(&nft_net->commit_mutex);
10167         nft_net->base_seq = 1;
10168         nft_net->validate_state = NFT_VALIDATE_SKIP;
10169
10170         return 0;
10171 }
10172
10173 static void __net_exit nf_tables_pre_exit_net(struct net *net)
10174 {
10175         struct nftables_pernet *nft_net = nft_pernet(net);
10176
10177         mutex_lock(&nft_net->commit_mutex);
10178         __nft_release_hooks(net);
10179         mutex_unlock(&nft_net->commit_mutex);
10180 }
10181
10182 static void __net_exit nf_tables_exit_net(struct net *net)
10183 {
10184         struct nftables_pernet *nft_net = nft_pernet(net);
10185
10186         mutex_lock(&nft_net->commit_mutex);
10187         if (!list_empty(&nft_net->commit_list) ||
10188             !list_empty(&nft_net->module_list))
10189                 __nf_tables_abort(net, NFNL_ABORT_NONE);
10190         __nft_release_tables(net);
10191         mutex_unlock(&nft_net->commit_mutex);
10192         WARN_ON_ONCE(!list_empty(&nft_net->tables));
10193         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
10194         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10195 }
10196
10197 static struct pernet_operations nf_tables_net_ops = {
10198         .init           = nf_tables_init_net,
10199         .pre_exit       = nf_tables_pre_exit_net,
10200         .exit           = nf_tables_exit_net,
10201         .id             = &nf_tables_net_id,
10202         .size           = sizeof(struct nftables_pernet),
10203 };
10204
10205 static int __init nf_tables_module_init(void)
10206 {
10207         int err;
10208
10209         err = register_pernet_subsys(&nf_tables_net_ops);
10210         if (err < 0)
10211                 return err;
10212
10213         err = nft_chain_filter_init();
10214         if (err < 0)
10215                 goto err_chain_filter;
10216
10217         err = nf_tables_core_module_init();
10218         if (err < 0)
10219                 goto err_core_module;
10220
10221         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
10222         if (err < 0)
10223                 goto err_netdev_notifier;
10224
10225         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
10226         if (err < 0)
10227                 goto err_rht_objname;
10228
10229         err = nft_offload_init();
10230         if (err < 0)
10231                 goto err_offload;
10232
10233         err = netlink_register_notifier(&nft_nl_notifier);
10234         if (err < 0)
10235                 goto err_netlink_notifier;
10236
10237         /* must be last */
10238         err = nfnetlink_subsys_register(&nf_tables_subsys);
10239         if (err < 0)
10240                 goto err_nfnl_subsys;
10241
10242         nft_chain_route_init();
10243
10244         return err;
10245
10246 err_nfnl_subsys:
10247         netlink_unregister_notifier(&nft_nl_notifier);
10248 err_netlink_notifier:
10249         nft_offload_exit();
10250 err_offload:
10251         rhltable_destroy(&nft_objname_ht);
10252 err_rht_objname:
10253         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10254 err_netdev_notifier:
10255         nf_tables_core_module_exit();
10256 err_core_module:
10257         nft_chain_filter_fini();
10258 err_chain_filter:
10259         unregister_pernet_subsys(&nf_tables_net_ops);
10260         return err;
10261 }
10262
10263 static void __exit nf_tables_module_exit(void)
10264 {
10265         nfnetlink_subsys_unregister(&nf_tables_subsys);
10266         netlink_unregister_notifier(&nft_nl_notifier);
10267         nft_offload_exit();
10268         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
10269         nft_chain_filter_fini();
10270         nft_chain_route_fini();
10271         unregister_pernet_subsys(&nf_tables_net_ops);
10272         cancel_work_sync(&trans_destroy_work);
10273         rcu_barrier();
10274         rhltable_destroy(&nft_objname_ht);
10275         nf_tables_core_module_exit();
10276 }
10277
10278 module_init(nf_tables_module_init);
10279 module_exit(nf_tables_module_exit);
10280
10281 MODULE_LICENSE("GPL");
10282 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
10283 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);