net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27 #define NFT_SET_MAX_ANONLEN 16
  28
  29 unsigned int nf_tables_net_id __read_mostly;
  30
  31 static LIST_HEAD(nf_tables_expressions);
  32 static LIST_HEAD(nf_tables_objects);
  33 static LIST_HEAD(nf_tables_flowtables);
  34 static LIST_HEAD(nf_tables_destroy_list);
  35 static LIST_HEAD(nf_tables_gc_list);
  36 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  37 static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
  38
  39 enum {
  40         NFT_VALIDATE_SKIP       = 0,
  41         NFT_VALIDATE_NEED,
  42         NFT_VALIDATE_DO,
  43 };
  44
  45 static struct rhltable nft_objname_ht;
  46
  47 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  48 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  49 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  50
  51 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  52 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  53 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  54
  55 static const struct rhashtable_params nft_chain_ht_params = {
  56         .head_offset            = offsetof(struct nft_chain, rhlhead),
  57         .key_offset             = offsetof(struct nft_chain, name),
  58         .hashfn                 = nft_chain_hash,
  59         .obj_hashfn             = nft_chain_hash_obj,
  60         .obj_cmpfn              = nft_chain_hash_cmp,
  61         .automatic_shrinking    = true,
  62 };
  63
  64 static const struct rhashtable_params nft_objname_ht_params = {
  65         .head_offset            = offsetof(struct nft_object, rhlhead),
  66         .key_offset             = offsetof(struct nft_object, key),
  67         .hashfn                 = nft_objname_hash,
  68         .obj_hashfn             = nft_objname_hash_obj,
  69         .obj_cmpfn              = nft_objname_hash_cmp,
  70         .automatic_shrinking    = true,
  71 };
  72
  73 struct nft_audit_data {
  74         struct nft_table *table;
  75         int entries;
  76         int op;
  77         struct list_head list;
  78 };
  79
  80 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  81         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  82         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  83         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  84         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  85         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  87         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  88         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  90         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  91         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  93         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  94         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  96         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  97         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  99         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
 100         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
 101         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
 102         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 103         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 104         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 105         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 106         [NFT_MSG_GETSETELEM_RESET] = AUDIT_NFT_OP_SETELEM_RESET,
 107 };
 108
 109 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 110 {
 111         switch (table->validate_state) {
 112         case NFT_VALIDATE_SKIP:
 113                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 114                 break;
 115         case NFT_VALIDATE_NEED:
 116                 break;
 117         case NFT_VALIDATE_DO:
 118                 if (new_validate_state == NFT_VALIDATE_NEED)
 119                         return;
 120         }
 121
 122         table->validate_state = new_validate_state;
 123 }
 124 static void nf_tables_trans_destroy_work(struct work_struct *w);
 125 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 126
 127 static void nft_trans_gc_work(struct work_struct *work);
 128 static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
 129
 130 static void nft_ctx_init(struct nft_ctx *ctx,
 131                          struct net *net,
 132                          const struct sk_buff *skb,
 133                          const struct nlmsghdr *nlh,
 134                          u8 family,
 135                          struct nft_table *table,
 136                          struct nft_chain *chain,
 137                          const struct nlattr * const *nla)
 138 {
 139         ctx->net        = net;
 140         ctx->family     = family;
 141         ctx->level      = 0;
 142         ctx->table      = table;
 143         ctx->chain      = chain;
 144         ctx->nla        = nla;
 145         ctx->portid     = NETLINK_CB(skb).portid;
 146         ctx->report     = nlmsg_report(nlh);
 147         ctx->flags      = nlh->nlmsg_flags;
 148         ctx->seq        = nlh->nlmsg_seq;
 149 }
 150
 151 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 152                                              int msg_type, u32 size, gfp_t gfp)
 153 {
 154         struct nft_trans *trans;
 155
 156         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 157         if (trans == NULL)
 158                 return NULL;
 159
 160         INIT_LIST_HEAD(&trans->list);
 161         INIT_LIST_HEAD(&trans->binding_list);
 162         trans->msg_type = msg_type;
 163         trans->ctx      = *ctx;
 164
 165         return trans;
 166 }
 167
 168 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 169                                          int msg_type, u32 size)
 170 {
 171         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 172 }
 173
 174 static void nft_trans_list_del(struct nft_trans *trans)
 175 {
 176         list_del(&trans->list);
 177         list_del(&trans->binding_list);
 178 }
 179
 180 static void nft_trans_destroy(struct nft_trans *trans)
 181 {
 182         nft_trans_list_del(trans);
 183         kfree(trans);
 184 }
 185
 186 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 187                                  bool bind)
 188 {
 189         struct nftables_pernet *nft_net;
 190         struct net *net = ctx->net;
 191         struct nft_trans *trans;
 192
 193         if (!nft_set_is_anonymous(set))
 194                 return;
 195
 196         nft_net = nft_pernet(net);
 197         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 198                 switch (trans->msg_type) {
 199                 case NFT_MSG_NEWSET:
 200                         if (nft_trans_set(trans) == set)
 201                                 nft_trans_set_bound(trans) = bind;
 202                         break;
 203                 case NFT_MSG_NEWSETELEM:
 204                         if (nft_trans_elem_set(trans) == set)
 205                                 nft_trans_elem_set_bound(trans) = bind;
 206                         break;
 207                 }
 208         }
 209 }
 210
 211 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 212 {
 213         return __nft_set_trans_bind(ctx, set, true);
 214 }
 215
 216 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 217 {
 218         return __nft_set_trans_bind(ctx, set, false);
 219 }
 220
 221 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 222                                    struct nft_chain *chain, bool bind)
 223 {
 224         struct nftables_pernet *nft_net;
 225         struct net *net = ctx->net;
 226         struct nft_trans *trans;
 227
 228         if (!nft_chain_binding(chain))
 229                 return;
 230
 231         nft_net = nft_pernet(net);
 232         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 233                 switch (trans->msg_type) {
 234                 case NFT_MSG_NEWCHAIN:
 235                         if (nft_trans_chain(trans) == chain)
 236                                 nft_trans_chain_bound(trans) = bind;
 237                         break;
 238                 case NFT_MSG_NEWRULE:
 239                         if (trans->ctx.chain == chain)
 240                                 nft_trans_rule_bound(trans) = bind;
 241                         break;
 242                 }
 243         }
 244 }
 245
 246 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 247                                  struct nft_chain *chain)
 248 {
 249         __nft_chain_trans_bind(ctx, chain, true);
 250 }
 251
 252 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 253 {
 254         if (!nft_chain_binding(chain))
 255                 return 0;
 256
 257         if (nft_chain_binding(ctx->chain))
 258                 return -EOPNOTSUPP;
 259
 260         if (chain->bound)
 261                 return -EBUSY;
 262
 263         if (!nft_use_inc(&chain->use))
 264                 return -EMFILE;
 265
 266         chain->bound = true;
 267         nft_chain_trans_bind(ctx, chain);
 268
 269         return 0;
 270 }
 271
 272 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 273 {
 274         __nft_chain_trans_bind(ctx, chain, false);
 275 }
 276
 277 static int nft_netdev_register_hooks(struct net *net,
 278                                      struct list_head *hook_list)
 279 {
 280         struct nft_hook *hook;
 281         int err, j;
 282
 283         j = 0;
 284         list_for_each_entry(hook, hook_list, list) {
 285                 err = nf_register_net_hook(net, &hook->ops);
 286                 if (err < 0)
 287                         goto err_register;
 288
 289                 j++;
 290         }
 291         return 0;
 292
 293 err_register:
 294         list_for_each_entry(hook, hook_list, list) {
 295                 if (j-- <= 0)
 296                         break;
 297
 298                 nf_unregister_net_hook(net, &hook->ops);
 299         }
 300         return err;
 301 }
 302
 303 static void nft_netdev_unregister_hooks(struct net *net,
 304                                         struct list_head *hook_list,
 305                                         bool release_netdev)
 306 {
 307         struct nft_hook *hook, *next;
 308
 309         list_for_each_entry_safe(hook, next, hook_list, list) {
 310                 nf_unregister_net_hook(net, &hook->ops);
 311                 if (release_netdev) {
 312                         list_del(&hook->list);
 313                         kfree_rcu(hook, rcu);
 314                 }
 315         }
 316 }
 317
 318 static int nf_tables_register_hook(struct net *net,
 319                                    const struct nft_table *table,
 320                                    struct nft_chain *chain)
 321 {
 322         struct nft_base_chain *basechain;
 323         const struct nf_hook_ops *ops;
 324
 325         if (table->flags & NFT_TABLE_F_DORMANT ||
 326             !nft_is_base_chain(chain))
 327                 return 0;
 328
 329         basechain = nft_base_chain(chain);
 330         ops = &basechain->ops;
 331
 332         if (basechain->type->ops_register)
 333                 return basechain->type->ops_register(net, ops);
 334
 335         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 336                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 337
 338         return nf_register_net_hook(net, &basechain->ops);
 339 }
 340
 341 static void __nf_tables_unregister_hook(struct net *net,
 342                                         const struct nft_table *table,
 343                                         struct nft_chain *chain,
 344                                         bool release_netdev)
 345 {
 346         struct nft_base_chain *basechain;
 347         const struct nf_hook_ops *ops;
 348
 349         if (table->flags & NFT_TABLE_F_DORMANT ||
 350             !nft_is_base_chain(chain))
 351                 return;
 352         basechain = nft_base_chain(chain);
 353         ops = &basechain->ops;
 354
 355         if (basechain->type->ops_unregister)
 356                 return basechain->type->ops_unregister(net, ops);
 357
 358         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 359                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 360                                             release_netdev);
 361         else
 362                 nf_unregister_net_hook(net, &basechain->ops);
 363 }
 364
 365 static void nf_tables_unregister_hook(struct net *net,
 366                                       const struct nft_table *table,
 367                                       struct nft_chain *chain)
 368 {
 369         return __nf_tables_unregister_hook(net, table, chain, false);
 370 }
 371
 372 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 373 {
 374         struct nftables_pernet *nft_net = nft_pernet(net);
 375
 376         switch (trans->msg_type) {
 377         case NFT_MSG_NEWSET:
 378                 if (!nft_trans_set_update(trans) &&
 379                     nft_set_is_anonymous(nft_trans_set(trans)))
 380                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 381                 break;
 382         case NFT_MSG_NEWCHAIN:
 383                 if (!nft_trans_chain_update(trans) &&
 384                     nft_chain_binding(nft_trans_chain(trans)))
 385                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 386                 break;
 387         }
 388
 389         list_add_tail(&trans->list, &nft_net->commit_list);
 390 }
 391
 392 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 393 {
 394         struct nft_trans *trans;
 395
 396         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 397         if (trans == NULL)
 398                 return -ENOMEM;
 399
 400         if (msg_type == NFT_MSG_NEWTABLE)
 401                 nft_activate_next(ctx->net, ctx->table);
 402
 403         nft_trans_commit_list_add_tail(ctx->net, trans);
 404         return 0;
 405 }
 406
 407 static int nft_deltable(struct nft_ctx *ctx)
 408 {
 409         int err;
 410
 411         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 412         if (err < 0)
 413                 return err;
 414
 415         nft_deactivate_next(ctx->net, ctx->table);
 416         return err;
 417 }
 418
 419 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 420 {
 421         struct nft_trans *trans;
 422
 423         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 424         if (trans == NULL)
 425                 return ERR_PTR(-ENOMEM);
 426
 427         if (msg_type == NFT_MSG_NEWCHAIN) {
 428                 nft_activate_next(ctx->net, ctx->chain);
 429
 430                 if (ctx->nla[NFTA_CHAIN_ID]) {
 431                         nft_trans_chain_id(trans) =
 432                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 433                 }
 434         }
 435         nft_trans_chain(trans) = ctx->chain;
 436         nft_trans_commit_list_add_tail(ctx->net, trans);
 437
 438         return trans;
 439 }
 440
 441 static int nft_delchain(struct nft_ctx *ctx)
 442 {
 443         struct nft_trans *trans;
 444
 445         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 446         if (IS_ERR(trans))
 447                 return PTR_ERR(trans);
 448
 449         nft_use_dec(&ctx->table->use);
 450         nft_deactivate_next(ctx->net, ctx->chain);
 451
 452         return 0;
 453 }
 454
 455 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 456 {
 457         struct nft_expr *expr;
 458
 459         expr = nft_expr_first(rule);
 460         while (nft_expr_more(rule, expr)) {
 461                 if (expr->ops->activate)
 462                         expr->ops->activate(ctx, expr);
 463
 464                 expr = nft_expr_next(expr);
 465         }
 466 }
 467
 468 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 469                               enum nft_trans_phase phase)
 470 {
 471         struct nft_expr *expr;
 472
 473         expr = nft_expr_first(rule);
 474         while (nft_expr_more(rule, expr)) {
 475                 if (expr->ops->deactivate)
 476                         expr->ops->deactivate(ctx, expr, phase);
 477
 478                 expr = nft_expr_next(expr);
 479         }
 480 }
 481
 482 static int
 483 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 484 {
 485         /* You cannot delete the same rule twice */
 486         if (nft_is_active_next(ctx->net, rule)) {
 487                 nft_deactivate_next(ctx->net, rule);
 488                 nft_use_dec(&ctx->chain->use);
 489                 return 0;
 490         }
 491         return -ENOENT;
 492 }
 493
 494 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 495                                             struct nft_rule *rule)
 496 {
 497         struct nft_trans *trans;
 498
 499         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 500         if (trans == NULL)
 501                 return NULL;
 502
 503         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 504                 nft_trans_rule_id(trans) =
 505                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 506         }
 507         nft_trans_rule(trans) = rule;
 508         nft_trans_commit_list_add_tail(ctx->net, trans);
 509
 510         return trans;
 511 }
 512
 513 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 514 {
 515         struct nft_flow_rule *flow;
 516         struct nft_trans *trans;
 517         int err;
 518
 519         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 520         if (trans == NULL)
 521                 return -ENOMEM;
 522
 523         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 524                 flow = nft_flow_rule_create(ctx->net, rule);
 525                 if (IS_ERR(flow)) {
 526                         nft_trans_destroy(trans);
 527                         return PTR_ERR(flow);
 528                 }
 529
 530                 nft_trans_flow_rule(trans) = flow;
 531         }
 532
 533         err = nf_tables_delrule_deactivate(ctx, rule);
 534         if (err < 0) {
 535                 nft_trans_destroy(trans);
 536                 return err;
 537         }
 538         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 539
 540         return 0;
 541 }
 542
 543 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 544 {
 545         struct nft_rule *rule;
 546         int err;
 547
 548         list_for_each_entry(rule, &ctx->chain->rules, list) {
 549                 if (!nft_is_active_next(ctx->net, rule))
 550                         continue;
 551
 552                 err = nft_delrule(ctx, rule);
 553                 if (err < 0)
 554                         return err;
 555         }
 556         return 0;
 557 }
 558
 559 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 560                                struct nft_set *set,
 561                                const struct nft_set_desc *desc)
 562 {
 563         struct nft_trans *trans;
 564
 565         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 566         if (trans == NULL)
 567                 return -ENOMEM;
 568
 569         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 570                 nft_trans_set_id(trans) =
 571                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 572                 nft_activate_next(ctx->net, set);
 573         }
 574         nft_trans_set(trans) = set;
 575         if (desc) {
 576                 nft_trans_set_update(trans) = true;
 577                 nft_trans_set_gc_int(trans) = desc->gc_int;
 578                 nft_trans_set_timeout(trans) = desc->timeout;
 579                 nft_trans_set_size(trans) = desc->size;
 580         }
 581         nft_trans_commit_list_add_tail(ctx->net, trans);
 582
 583         return 0;
 584 }
 585
 586 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 587                              struct nft_set *set)
 588 {
 589         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 590 }
 591
 592 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
 593                                   struct nft_set *set,
 594                                   const struct nft_set_iter *iter,
 595                                   struct nft_elem_priv *elem_priv)
 596 {
 597         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 598
 599         if (!nft_set_elem_active(ext, iter->genmask))
 600                 return 0;
 601
 602         nft_set_elem_change_active(ctx->net, set, ext);
 603         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
 604
 605         return 0;
 606 }
 607
 608 struct nft_set_elem_catchall {
 609         struct list_head        list;
 610         struct rcu_head         rcu;
 611         struct nft_elem_priv    *elem;
 612 };
 613
 614 static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
 615                                         struct nft_set *set)
 616 {
 617         u8 genmask = nft_genmask_next(ctx->net);
 618         struct nft_set_elem_catchall *catchall;
 619         struct nft_set_ext *ext;
 620
 621         list_for_each_entry(catchall, &set->catchall_list, list) {
 622                 ext = nft_set_elem_ext(set, catchall->elem);
 623                 if (!nft_set_elem_active(ext, genmask))
 624                         continue;
 625
 626                 nft_set_elem_change_active(ctx->net, set, ext);
 627                 nft_setelem_data_deactivate(ctx->net, set, catchall->elem);
 628                 break;
 629         }
 630 }
 631
 632 static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
 633 {
 634         struct nft_set_iter iter = {
 635                 .genmask        = nft_genmask_next(ctx->net),
 636                 .type           = NFT_ITER_UPDATE,
 637                 .fn             = nft_mapelem_deactivate,
 638         };
 639
 640         set->ops->walk(ctx, set, &iter);
 641         WARN_ON_ONCE(iter.err);
 642
 643         nft_map_catchall_deactivate(ctx, set);
 644 }
 645
 646 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 647 {
 648         int err;
 649
 650         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 651         if (err < 0)
 652                 return err;
 653
 654         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 655                 nft_map_deactivate(ctx, set);
 656
 657         nft_deactivate_next(ctx->net, set);
 658         nft_use_dec(&ctx->table->use);
 659
 660         return err;
 661 }
 662
 663 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 664                              struct nft_object *obj)
 665 {
 666         struct nft_trans *trans;
 667
 668         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 669         if (trans == NULL)
 670                 return -ENOMEM;
 671
 672         if (msg_type == NFT_MSG_NEWOBJ)
 673                 nft_activate_next(ctx->net, obj);
 674
 675         nft_trans_obj(trans) = obj;
 676         nft_trans_commit_list_add_tail(ctx->net, trans);
 677
 678         return 0;
 679 }
 680
 681 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 682 {
 683         int err;
 684
 685         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 686         if (err < 0)
 687                 return err;
 688
 689         nft_deactivate_next(ctx->net, obj);
 690         nft_use_dec(&ctx->table->use);
 691
 692         return err;
 693 }
 694
 695 static struct nft_trans *
 696 nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 697                         struct nft_flowtable *flowtable)
 698 {
 699         struct nft_trans *trans;
 700
 701         trans = nft_trans_alloc(ctx, msg_type,
 702                                 sizeof(struct nft_trans_flowtable));
 703         if (trans == NULL)
 704                 return ERR_PTR(-ENOMEM);
 705
 706         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 707                 nft_activate_next(ctx->net, flowtable);
 708
 709         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 710         nft_trans_flowtable(trans) = flowtable;
 711         nft_trans_commit_list_add_tail(ctx->net, trans);
 712
 713         return trans;
 714 }
 715
 716 static int nft_delflowtable(struct nft_ctx *ctx,
 717                             struct nft_flowtable *flowtable)
 718 {
 719         struct nft_trans *trans;
 720
 721         trans = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 722         if (IS_ERR(trans))
 723                 return PTR_ERR(trans);
 724
 725         nft_deactivate_next(ctx->net, flowtable);
 726         nft_use_dec(&ctx->table->use);
 727
 728         return 0;
 729 }
 730
 731 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 732 {
 733         int i;
 734
 735         for (i = track->regs[dreg].num_reg; i > 0; i--)
 736                 __nft_reg_track_cancel(track, dreg - i);
 737 }
 738
 739 static void __nft_reg_track_update(struct nft_regs_track *track,
 740                                    const struct nft_expr *expr,
 741                                    u8 dreg, u8 num_reg)
 742 {
 743         track->regs[dreg].selector = expr;
 744         track->regs[dreg].bitwise = NULL;
 745         track->regs[dreg].num_reg = num_reg;
 746 }
 747
 748 void nft_reg_track_update(struct nft_regs_track *track,
 749                           const struct nft_expr *expr, u8 dreg, u8 len)
 750 {
 751         unsigned int regcount;
 752         int i;
 753
 754         __nft_reg_track_clobber(track, dreg);
 755
 756         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 757         for (i = 0; i < regcount; i++, dreg++)
 758                 __nft_reg_track_update(track, expr, dreg, i);
 759 }
 760 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 761
 762 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 763 {
 764         unsigned int regcount;
 765         int i;
 766
 767         __nft_reg_track_clobber(track, dreg);
 768
 769         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 770         for (i = 0; i < regcount; i++, dreg++)
 771                 __nft_reg_track_cancel(track, dreg);
 772 }
 773 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 774
 775 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 776 {
 777         track->regs[dreg].selector = NULL;
 778         track->regs[dreg].bitwise = NULL;
 779         track->regs[dreg].num_reg = 0;
 780 }
 781 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 782
 783 /*
 784  * Tables
 785  */
 786
 787 static struct nft_table *nft_table_lookup(const struct net *net,
 788                                           const struct nlattr *nla,
 789                                           u8 family, u8 genmask, u32 nlpid)
 790 {
 791         struct nftables_pernet *nft_net;
 792         struct nft_table *table;
 793
 794         if (nla == NULL)
 795                 return ERR_PTR(-EINVAL);
 796
 797         nft_net = nft_pernet(net);
 798         list_for_each_entry_rcu(table, &nft_net->tables, list,
 799                                 lockdep_is_held(&nft_net->commit_mutex)) {
 800                 if (!nla_strcmp(nla, table->name) &&
 801                     table->family == family &&
 802                     nft_active_genmask(table, genmask)) {
 803                         if (nft_table_has_owner(table) &&
 804                             nlpid && table->nlpid != nlpid)
 805                                 return ERR_PTR(-EPERM);
 806
 807                         return table;
 808                 }
 809         }
 810
 811         return ERR_PTR(-ENOENT);
 812 }
 813
 814 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 815                                                    const struct nlattr *nla,
 816                                                    int family, u8 genmask, u32 nlpid)
 817 {
 818         struct nftables_pernet *nft_net;
 819         struct nft_table *table;
 820
 821         nft_net = nft_pernet(net);
 822         list_for_each_entry(table, &nft_net->tables, list) {
 823                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 824                     table->family == family &&
 825                     nft_active_genmask(table, genmask)) {
 826                         if (nft_table_has_owner(table) &&
 827                             nlpid && table->nlpid != nlpid)
 828                                 return ERR_PTR(-EPERM);
 829
 830                         return table;
 831                 }
 832         }
 833
 834         return ERR_PTR(-ENOENT);
 835 }
 836
 837 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 838 {
 839         return ++table->hgenerator;
 840 }
 841
 842 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 843
 844 static const struct nft_chain_type *
 845 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 846 {
 847         if (family >= NFPROTO_NUMPROTO ||
 848             type >= NFT_CHAIN_T_MAX)
 849                 return NULL;
 850
 851         return chain_type[family][type];
 852 }
 853
 854 static const struct nft_chain_type *
 855 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 856 {
 857         const struct nft_chain_type *type;
 858         int i;
 859
 860         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 861                 type = __nft_chain_type_get(family, i);
 862                 if (!type)
 863                         continue;
 864                 if (!nla_strcmp(nla, type->name))
 865                         return type;
 866         }
 867         return NULL;
 868 }
 869
 870 struct nft_module_request {
 871         struct list_head        list;
 872         char                    module[MODULE_NAME_LEN];
 873         bool                    done;
 874 };
 875
 876 #ifdef CONFIG_MODULES
 877 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 878                                       ...)
 879 {
 880         char module_name[MODULE_NAME_LEN];
 881         struct nftables_pernet *nft_net;
 882         struct nft_module_request *req;
 883         va_list args;
 884         int ret;
 885
 886         va_start(args, fmt);
 887         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 888         va_end(args);
 889         if (ret >= MODULE_NAME_LEN)
 890                 return 0;
 891
 892         nft_net = nft_pernet(net);
 893         list_for_each_entry(req, &nft_net->module_list, list) {
 894                 if (!strcmp(req->module, module_name)) {
 895                         if (req->done)
 896                                 return 0;
 897
 898                         /* A request to load this module already exists. */
 899                         return -EAGAIN;
 900                 }
 901         }
 902
 903         req = kmalloc(sizeof(*req), GFP_KERNEL);
 904         if (!req)
 905                 return -ENOMEM;
 906
 907         req->done = false;
 908         strscpy(req->module, module_name, MODULE_NAME_LEN);
 909         list_add_tail(&req->list, &nft_net->module_list);
 910
 911         return -EAGAIN;
 912 }
 913 EXPORT_SYMBOL_GPL(nft_request_module);
 914 #endif
 915
 916 static void lockdep_nfnl_nft_mutex_not_held(void)
 917 {
 918 #ifdef CONFIG_PROVE_LOCKING
 919         if (debug_locks)
 920                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 921 #endif
 922 }
 923
 924 static const struct nft_chain_type *
 925 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 926                             u8 family, bool autoload)
 927 {
 928         const struct nft_chain_type *type;
 929
 930         type = __nf_tables_chain_type_lookup(nla, family);
 931         if (type != NULL)
 932                 return type;
 933
 934         lockdep_nfnl_nft_mutex_not_held();
 935 #ifdef CONFIG_MODULES
 936         if (autoload) {
 937                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 938                                        nla_len(nla),
 939                                        (const char *)nla_data(nla)) == -EAGAIN)
 940                         return ERR_PTR(-EAGAIN);
 941         }
 942 #endif
 943         return ERR_PTR(-ENOENT);
 944 }
 945
 946 static __be16 nft_base_seq(const struct net *net)
 947 {
 948         struct nftables_pernet *nft_net = nft_pernet(net);
 949
 950         return htons(nft_net->base_seq & 0xffff);
 951 }
 952
 953 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 954         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 955                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 956         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 957         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 958         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 959                                     .len = NFT_USERDATA_MAXLEN }
 960 };
 961
 962 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 963                                      u32 portid, u32 seq, int event, u32 flags,
 964                                      int family, const struct nft_table *table)
 965 {
 966         struct nlmsghdr *nlh;
 967
 968         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 969         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 970                            NFNETLINK_V0, nft_base_seq(net));
 971         if (!nlh)
 972                 goto nla_put_failure;
 973
 974         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 975             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 976             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 977                          NFTA_TABLE_PAD))
 978                 goto nla_put_failure;
 979
 980         if (event == NFT_MSG_DELTABLE) {
 981                 nlmsg_end(skb, nlh);
 982                 return 0;
 983         }
 984
 985         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 986                          htonl(table->flags & NFT_TABLE_F_MASK)))
 987                 goto nla_put_failure;
 988
 989         if (nft_table_has_owner(table) &&
 990             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 991                 goto nla_put_failure;
 992
 993         if (table->udata) {
 994                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 995                         goto nla_put_failure;
 996         }
 997
 998         nlmsg_end(skb, nlh);
 999         return 0;
1000
1001 nla_put_failure:
1002         nlmsg_trim(skb, nlh);
1003         return -1;
1004 }
1005
1006 struct nftnl_skb_parms {
1007         bool report;
1008 };
1009 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
1010
1011 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
1012                                struct list_head *notify_list)
1013 {
1014         NFT_CB(skb).report = report;
1015         list_add_tail(&skb->list, notify_list);
1016 }
1017
1018 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
1019 {
1020         struct nftables_pernet *nft_net;
1021         struct sk_buff *skb;
1022         u16 flags = 0;
1023         int err;
1024
1025         if (!ctx->report &&
1026             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1027                 return;
1028
1029         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1030         if (skb == NULL)
1031                 goto err;
1032
1033         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1034                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1035
1036         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
1037                                         event, flags, ctx->family, ctx->table);
1038         if (err < 0) {
1039                 kfree_skb(skb);
1040                 goto err;
1041         }
1042
1043         nft_net = nft_pernet(ctx->net);
1044         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1045         return;
1046 err:
1047         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1048 }
1049
1050 static int nf_tables_dump_tables(struct sk_buff *skb,
1051                                  struct netlink_callback *cb)
1052 {
1053         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1054         struct nftables_pernet *nft_net;
1055         const struct nft_table *table;
1056         unsigned int idx = 0, s_idx = cb->args[0];
1057         struct net *net = sock_net(skb->sk);
1058         int family = nfmsg->nfgen_family;
1059
1060         rcu_read_lock();
1061         nft_net = nft_pernet(net);
1062         cb->seq = READ_ONCE(nft_net->base_seq);
1063
1064         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1065                 if (family != NFPROTO_UNSPEC && family != table->family)
1066                         continue;
1067
1068                 if (idx < s_idx)
1069                         goto cont;
1070                 if (idx > s_idx)
1071                         memset(&cb->args[1], 0,
1072                                sizeof(cb->args) - sizeof(cb->args[0]));
1073                 if (!nft_is_active(net, table))
1074                         continue;
1075                 if (nf_tables_fill_table_info(skb, net,
1076                                               NETLINK_CB(cb->skb).portid,
1077                                               cb->nlh->nlmsg_seq,
1078                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
1079                                               table->family, table) < 0)
1080                         goto done;
1081
1082                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1083 cont:
1084                 idx++;
1085         }
1086 done:
1087         rcu_read_unlock();
1088         cb->args[0] = idx;
1089         return skb->len;
1090 }
1091
1092 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1093                                       const struct nlmsghdr *nlh,
1094                                       struct netlink_dump_control *c)
1095 {
1096         int err;
1097
1098         if (!try_module_get(THIS_MODULE))
1099                 return -EINVAL;
1100
1101         rcu_read_unlock();
1102         err = netlink_dump_start(nlsk, skb, nlh, c);
1103         rcu_read_lock();
1104         module_put(THIS_MODULE);
1105
1106         return err;
1107 }
1108
1109 /* called with rcu_read_lock held */
1110 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1111                               const struct nlattr * const nla[])
1112 {
1113         struct netlink_ext_ack *extack = info->extack;
1114         u8 genmask = nft_genmask_cur(info->net);
1115         u8 family = info->nfmsg->nfgen_family;
1116         const struct nft_table *table;
1117         struct net *net = info->net;
1118         struct sk_buff *skb2;
1119         int err;
1120
1121         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1122                 struct netlink_dump_control c = {
1123                         .dump = nf_tables_dump_tables,
1124                         .module = THIS_MODULE,
1125                 };
1126
1127                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1128         }
1129
1130         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1131         if (IS_ERR(table)) {
1132                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1133                 return PTR_ERR(table);
1134         }
1135
1136         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1137         if (!skb2)
1138                 return -ENOMEM;
1139
1140         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1141                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1142                                         0, family, table);
1143         if (err < 0)
1144                 goto err_fill_table_info;
1145
1146         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1147
1148 err_fill_table_info:
1149         kfree_skb(skb2);
1150         return err;
1151 }
1152
1153 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1154 {
1155         struct nft_chain *chain;
1156         u32 i = 0;
1157
1158         list_for_each_entry(chain, &table->chains, list) {
1159                 if (!nft_is_active_next(net, chain))
1160                         continue;
1161                 if (!nft_is_base_chain(chain))
1162                         continue;
1163
1164                 if (cnt && i++ == cnt)
1165                         break;
1166
1167                 nf_tables_unregister_hook(net, table, chain);
1168         }
1169 }
1170
1171 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1172 {
1173         struct nft_chain *chain;
1174         int err, i = 0;
1175
1176         list_for_each_entry(chain, &table->chains, list) {
1177                 if (!nft_is_active_next(net, chain))
1178                         continue;
1179                 if (!nft_is_base_chain(chain))
1180                         continue;
1181
1182                 err = nf_tables_register_hook(net, table, chain);
1183                 if (err < 0)
1184                         goto err_register_hooks;
1185
1186                 i++;
1187         }
1188         return 0;
1189
1190 err_register_hooks:
1191         if (i)
1192                 nft_table_disable(net, table, i);
1193         return err;
1194 }
1195
1196 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1197 {
1198         table->flags &= ~NFT_TABLE_F_DORMANT;
1199         nft_table_disable(net, table, 0);
1200         table->flags |= NFT_TABLE_F_DORMANT;
1201 }
1202
1203 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1204 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1205 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1206 #define __NFT_TABLE_F_WAS_ORPHAN        (__NFT_TABLE_F_INTERNAL << 2)
1207 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1208                                          __NFT_TABLE_F_WAS_AWAKEN | \
1209                                          __NFT_TABLE_F_WAS_ORPHAN)
1210
1211 static bool nft_table_pending_update(const struct nft_ctx *ctx)
1212 {
1213         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1214         struct nft_trans *trans;
1215
1216         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
1217                 return true;
1218
1219         list_for_each_entry(trans, &nft_net->commit_list, list) {
1220                 if (trans->ctx.table == ctx->table &&
1221                     ((trans->msg_type == NFT_MSG_NEWCHAIN &&
1222                       nft_trans_chain_update(trans)) ||
1223                      (trans->msg_type == NFT_MSG_DELCHAIN &&
1224                       nft_is_base_chain(trans->ctx.chain))))
1225                         return true;
1226         }
1227
1228         return false;
1229 }
1230
1231 static int nf_tables_updtable(struct nft_ctx *ctx)
1232 {
1233         struct nft_trans *trans;
1234         u32 flags;
1235         int ret;
1236
1237         if (!ctx->nla[NFTA_TABLE_FLAGS])
1238                 return 0;
1239
1240         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1241         if (flags & ~NFT_TABLE_F_MASK)
1242                 return -EOPNOTSUPP;
1243
1244         if (flags == (ctx->table->flags & NFT_TABLE_F_MASK))
1245                 return 0;
1246
1247         if ((nft_table_has_owner(ctx->table) &&
1248              !(flags & NFT_TABLE_F_OWNER)) ||
1249             (flags & NFT_TABLE_F_OWNER &&
1250              !nft_table_is_orphan(ctx->table)))
1251                 return -EOPNOTSUPP;
1252
1253         if ((flags ^ ctx->table->flags) & NFT_TABLE_F_PERSIST)
1254                 return -EOPNOTSUPP;
1255
1256         /* No dormant off/on/off/on games in single transaction */
1257         if (nft_table_pending_update(ctx))
1258                 return -EINVAL;
1259
1260         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1261                                 sizeof(struct nft_trans_table));
1262         if (trans == NULL)
1263                 return -ENOMEM;
1264
1265         if ((flags & NFT_TABLE_F_DORMANT) &&
1266             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1267                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1268                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1269                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1270         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1271                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1272                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1273                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1274                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1275                         if (ret < 0)
1276                                 goto err_register_hooks;
1277
1278                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1279                 }
1280         }
1281
1282         if ((flags & NFT_TABLE_F_OWNER) &&
1283             !nft_table_has_owner(ctx->table)) {
1284                 ctx->table->nlpid = ctx->portid;
1285                 ctx->table->flags |= NFT_TABLE_F_OWNER |
1286                                      __NFT_TABLE_F_WAS_ORPHAN;
1287         }
1288
1289         nft_trans_table_update(trans) = true;
1290         nft_trans_commit_list_add_tail(ctx->net, trans);
1291
1292         return 0;
1293
1294 err_register_hooks:
1295         ctx->table->flags |= NFT_TABLE_F_DORMANT;
1296         nft_trans_destroy(trans);
1297         return ret;
1298 }
1299
1300 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1301 {
1302         const char *name = data;
1303
1304         return jhash(name, strlen(name), seed);
1305 }
1306
1307 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1308 {
1309         const struct nft_chain *chain = data;
1310
1311         return nft_chain_hash(chain->name, 0, seed);
1312 }
1313
1314 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1315                               const void *ptr)
1316 {
1317         const struct nft_chain *chain = ptr;
1318         const char *name = arg->key;
1319
1320         return strcmp(chain->name, name);
1321 }
1322
1323 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1324 {
1325         const struct nft_object_hash_key *k = data;
1326
1327         seed ^= hash_ptr(k->table, 32);
1328
1329         return jhash(k->name, strlen(k->name), seed);
1330 }
1331
1332 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1333 {
1334         const struct nft_object *obj = data;
1335
1336         return nft_objname_hash(&obj->key, 0, seed);
1337 }
1338
1339 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1340                                 const void *ptr)
1341 {
1342         const struct nft_object_hash_key *k = arg->key;
1343         const struct nft_object *obj = ptr;
1344
1345         if (obj->key.table != k->table)
1346                 return -1;
1347
1348         return strcmp(obj->key.name, k->name);
1349 }
1350
1351 static bool nft_supported_family(u8 family)
1352 {
1353         return false
1354 #ifdef CONFIG_NF_TABLES_INET
1355                 || family == NFPROTO_INET
1356 #endif
1357 #ifdef CONFIG_NF_TABLES_IPV4
1358                 || family == NFPROTO_IPV4
1359 #endif
1360 #ifdef CONFIG_NF_TABLES_ARP
1361                 || family == NFPROTO_ARP
1362 #endif
1363 #ifdef CONFIG_NF_TABLES_NETDEV
1364                 || family == NFPROTO_NETDEV
1365 #endif
1366 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1367                 || family == NFPROTO_BRIDGE
1368 #endif
1369 #ifdef CONFIG_NF_TABLES_IPV6
1370                 || family == NFPROTO_IPV6
1371 #endif
1372                 ;
1373 }
1374
1375 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1376                               const struct nlattr * const nla[])
1377 {
1378         struct nftables_pernet *nft_net = nft_pernet(info->net);
1379         struct netlink_ext_ack *extack = info->extack;
1380         u8 genmask = nft_genmask_next(info->net);
1381         u8 family = info->nfmsg->nfgen_family;
1382         struct net *net = info->net;
1383         const struct nlattr *attr;
1384         struct nft_table *table;
1385         struct nft_ctx ctx;
1386         u32 flags = 0;
1387         int err;
1388
1389         if (!nft_supported_family(family))
1390                 return -EOPNOTSUPP;
1391
1392         lockdep_assert_held(&nft_net->commit_mutex);
1393         attr = nla[NFTA_TABLE_NAME];
1394         table = nft_table_lookup(net, attr, family, genmask,
1395                                  NETLINK_CB(skb).portid);
1396         if (IS_ERR(table)) {
1397                 if (PTR_ERR(table) != -ENOENT)
1398                         return PTR_ERR(table);
1399         } else {
1400                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1401                         NL_SET_BAD_ATTR(extack, attr);
1402                         return -EEXIST;
1403                 }
1404                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1405                         return -EOPNOTSUPP;
1406
1407                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1408
1409                 return nf_tables_updtable(&ctx);
1410         }
1411
1412         if (nla[NFTA_TABLE_FLAGS]) {
1413                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1414                 if (flags & ~NFT_TABLE_F_MASK)
1415                         return -EOPNOTSUPP;
1416         }
1417
1418         err = -ENOMEM;
1419         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1420         if (table == NULL)
1421                 goto err_kzalloc;
1422
1423         table->validate_state = nft_net->validate_state;
1424         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1425         if (table->name == NULL)
1426                 goto err_strdup;
1427
1428         if (nla[NFTA_TABLE_USERDATA]) {
1429                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1430                 if (table->udata == NULL)
1431                         goto err_table_udata;
1432
1433                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1434         }
1435
1436         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1437         if (err)
1438                 goto err_chain_ht;
1439
1440         INIT_LIST_HEAD(&table->chains);
1441         INIT_LIST_HEAD(&table->sets);
1442         INIT_LIST_HEAD(&table->objects);
1443         INIT_LIST_HEAD(&table->flowtables);
1444         table->family = family;
1445         table->flags = flags;
1446         table->handle = ++nft_net->table_handle;
1447         if (table->flags & NFT_TABLE_F_OWNER)
1448                 table->nlpid = NETLINK_CB(skb).portid;
1449
1450         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1451         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1452         if (err < 0)
1453                 goto err_trans;
1454
1455         list_add_tail_rcu(&table->list, &nft_net->tables);
1456         return 0;
1457 err_trans:
1458         rhltable_destroy(&table->chains_ht);
1459 err_chain_ht:
1460         kfree(table->udata);
1461 err_table_udata:
1462         kfree(table->name);
1463 err_strdup:
1464         kfree(table);
1465 err_kzalloc:
1466         return err;
1467 }
1468
1469 static int nft_flush_table(struct nft_ctx *ctx)
1470 {
1471         struct nft_flowtable *flowtable, *nft;
1472         struct nft_chain *chain, *nc;
1473         struct nft_object *obj, *ne;
1474         struct nft_set *set, *ns;
1475         int err;
1476
1477         list_for_each_entry(chain, &ctx->table->chains, list) {
1478                 if (!nft_is_active_next(ctx->net, chain))
1479                         continue;
1480
1481                 if (nft_chain_binding(chain))
1482                         continue;
1483
1484                 ctx->chain = chain;
1485
1486                 err = nft_delrule_by_chain(ctx);
1487                 if (err < 0)
1488                         goto out;
1489         }
1490
1491         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1492                 if (!nft_is_active_next(ctx->net, set))
1493                         continue;
1494
1495                 if (nft_set_is_anonymous(set))
1496                         continue;
1497
1498                 err = nft_delset(ctx, set);
1499                 if (err < 0)
1500                         goto out;
1501         }
1502
1503         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1504                 if (!nft_is_active_next(ctx->net, flowtable))
1505                         continue;
1506
1507                 err = nft_delflowtable(ctx, flowtable);
1508                 if (err < 0)
1509                         goto out;
1510         }
1511
1512         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1513                 if (!nft_is_active_next(ctx->net, obj))
1514                         continue;
1515
1516                 err = nft_delobj(ctx, obj);
1517                 if (err < 0)
1518                         goto out;
1519         }
1520
1521         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1522                 if (!nft_is_active_next(ctx->net, chain))
1523                         continue;
1524
1525                 if (nft_chain_binding(chain))
1526                         continue;
1527
1528                 ctx->chain = chain;
1529
1530                 err = nft_delchain(ctx);
1531                 if (err < 0)
1532                         goto out;
1533         }
1534
1535         err = nft_deltable(ctx);
1536 out:
1537         return err;
1538 }
1539
1540 static int nft_flush(struct nft_ctx *ctx, int family)
1541 {
1542         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1543         const struct nlattr * const *nla = ctx->nla;
1544         struct nft_table *table, *nt;
1545         int err = 0;
1546
1547         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1548                 if (family != AF_UNSPEC && table->family != family)
1549                         continue;
1550
1551                 ctx->family = table->family;
1552
1553                 if (!nft_is_active_next(ctx->net, table))
1554                         continue;
1555
1556                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1557                         continue;
1558
1559                 if (nla[NFTA_TABLE_NAME] &&
1560                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1561                         continue;
1562
1563                 ctx->table = table;
1564
1565                 err = nft_flush_table(ctx);
1566                 if (err < 0)
1567                         goto out;
1568         }
1569 out:
1570         return err;
1571 }
1572
1573 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1574                               const struct nlattr * const nla[])
1575 {
1576         struct netlink_ext_ack *extack = info->extack;
1577         u8 genmask = nft_genmask_next(info->net);
1578         u8 family = info->nfmsg->nfgen_family;
1579         struct net *net = info->net;
1580         const struct nlattr *attr;
1581         struct nft_table *table;
1582         struct nft_ctx ctx;
1583
1584         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1585         if (family == AF_UNSPEC ||
1586             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1587                 return nft_flush(&ctx, family);
1588
1589         if (nla[NFTA_TABLE_HANDLE]) {
1590                 attr = nla[NFTA_TABLE_HANDLE];
1591                 table = nft_table_lookup_byhandle(net, attr, family, genmask,
1592                                                   NETLINK_CB(skb).portid);
1593         } else {
1594                 attr = nla[NFTA_TABLE_NAME];
1595                 table = nft_table_lookup(net, attr, family, genmask,
1596                                          NETLINK_CB(skb).portid);
1597         }
1598
1599         if (IS_ERR(table)) {
1600                 if (PTR_ERR(table) == -ENOENT &&
1601                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1602                         return 0;
1603
1604                 NL_SET_BAD_ATTR(extack, attr);
1605                 return PTR_ERR(table);
1606         }
1607
1608         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1609             table->use > 0)
1610                 return -EBUSY;
1611
1612         ctx.family = family;
1613         ctx.table = table;
1614
1615         return nft_flush_table(&ctx);
1616 }
1617
1618 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1619 {
1620         if (WARN_ON(ctx->table->use > 0))
1621                 return;
1622
1623         rhltable_destroy(&ctx->table->chains_ht);
1624         kfree(ctx->table->name);
1625         kfree(ctx->table->udata);
1626         kfree(ctx->table);
1627 }
1628
1629 void nft_register_chain_type(const struct nft_chain_type *ctype)
1630 {
1631         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1632         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1633                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1634                 return;
1635         }
1636         chain_type[ctype->family][ctype->type] = ctype;
1637         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1638 }
1639 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1640
1641 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1642 {
1643         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1644         chain_type[ctype->family][ctype->type] = NULL;
1645         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1646 }
1647 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1648
1649 /*
1650  * Chains
1651  */
1652
1653 static struct nft_chain *
1654 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1655 {
1656         struct nft_chain *chain;
1657
1658         list_for_each_entry(chain, &table->chains, list) {
1659                 if (chain->handle == handle &&
1660                     nft_active_genmask(chain, genmask))
1661                         return chain;
1662         }
1663
1664         return ERR_PTR(-ENOENT);
1665 }
1666
1667 static bool lockdep_commit_lock_is_held(const struct net *net)
1668 {
1669 #ifdef CONFIG_PROVE_LOCKING
1670         struct nftables_pernet *nft_net = nft_pernet(net);
1671
1672         return lockdep_is_held(&nft_net->commit_mutex);
1673 #else
1674         return true;
1675 #endif
1676 }
1677
1678 static struct nft_chain *nft_chain_lookup(struct net *net,
1679                                           struct nft_table *table,
1680                                           const struct nlattr *nla, u8 genmask)
1681 {
1682         char search[NFT_CHAIN_MAXNAMELEN + 1];
1683         struct rhlist_head *tmp, *list;
1684         struct nft_chain *chain;
1685
1686         if (nla == NULL)
1687                 return ERR_PTR(-EINVAL);
1688
1689         nla_strscpy(search, nla, sizeof(search));
1690
1691         WARN_ON(!rcu_read_lock_held() &&
1692                 !lockdep_commit_lock_is_held(net));
1693
1694         chain = ERR_PTR(-ENOENT);
1695         rcu_read_lock();
1696         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1697         if (!list)
1698                 goto out_unlock;
1699
1700         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1701                 if (nft_active_genmask(chain, genmask))
1702                         goto out_unlock;
1703         }
1704         chain = ERR_PTR(-ENOENT);
1705 out_unlock:
1706         rcu_read_unlock();
1707         return chain;
1708 }
1709
1710 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1711         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1712                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1713         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1714         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1715                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1716         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1717         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1718         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1719                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1720         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1721         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1722         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1723         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1724                                     .len = NFT_USERDATA_MAXLEN },
1725 };
1726
1727 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1728         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1729         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1730         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1731                                     .len = IFNAMSIZ - 1 },
1732 };
1733
1734 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1735 {
1736         struct nft_stats *cpu_stats, total;
1737         struct nlattr *nest;
1738         unsigned int seq;
1739         u64 pkts, bytes;
1740         int cpu;
1741
1742         if (!stats)
1743                 return 0;
1744
1745         memset(&total, 0, sizeof(total));
1746         for_each_possible_cpu(cpu) {
1747                 cpu_stats = per_cpu_ptr(stats, cpu);
1748                 do {
1749                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1750                         pkts = cpu_stats->pkts;
1751                         bytes = cpu_stats->bytes;
1752                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1753                 total.pkts += pkts;
1754                 total.bytes += bytes;
1755         }
1756         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1757         if (nest == NULL)
1758                 goto nla_put_failure;
1759
1760         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1761                          NFTA_COUNTER_PAD) ||
1762             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1763                          NFTA_COUNTER_PAD))
1764                 goto nla_put_failure;
1765
1766         nla_nest_end(skb, nest);
1767         return 0;
1768
1769 nla_put_failure:
1770         return -ENOSPC;
1771 }
1772
1773 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1774                                    const struct nft_base_chain *basechain,
1775                                    const struct list_head *hook_list)
1776 {
1777         const struct nf_hook_ops *ops = &basechain->ops;
1778         struct nft_hook *hook, *first = NULL;
1779         struct nlattr *nest, *nest_devs;
1780         int n = 0;
1781
1782         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1783         if (nest == NULL)
1784                 goto nla_put_failure;
1785         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1786                 goto nla_put_failure;
1787         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1788                 goto nla_put_failure;
1789
1790         if (nft_base_chain_netdev(family, ops->hooknum)) {
1791                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1792                 if (!nest_devs)
1793                         goto nla_put_failure;
1794
1795                 if (!hook_list)
1796                         hook_list = &basechain->hook_list;
1797
1798                 list_for_each_entry(hook, hook_list, list) {
1799                         if (!first)
1800                                 first = hook;
1801
1802                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1803                                            hook->ops.dev->name))
1804                                 goto nla_put_failure;
1805                         n++;
1806                 }
1807                 nla_nest_end(skb, nest_devs);
1808
1809                 if (n == 1 &&
1810                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1811                         goto nla_put_failure;
1812         }
1813         nla_nest_end(skb, nest);
1814
1815         return 0;
1816 nla_put_failure:
1817         return -1;
1818 }
1819
1820 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1821                                      u32 portid, u32 seq, int event, u32 flags,
1822                                      int family, const struct nft_table *table,
1823                                      const struct nft_chain *chain,
1824                                      const struct list_head *hook_list)
1825 {
1826         struct nlmsghdr *nlh;
1827
1828         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1829         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1830                            NFNETLINK_V0, nft_base_seq(net));
1831         if (!nlh)
1832                 goto nla_put_failure;
1833
1834         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1835             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1836             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1837                          NFTA_CHAIN_PAD))
1838                 goto nla_put_failure;
1839
1840         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1841                 nlmsg_end(skb, nlh);
1842                 return 0;
1843         }
1844
1845         if (nft_is_base_chain(chain)) {
1846                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1847                 struct nft_stats __percpu *stats;
1848
1849                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1850                         goto nla_put_failure;
1851
1852                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1853                                  htonl(basechain->policy)))
1854                         goto nla_put_failure;
1855
1856                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1857                         goto nla_put_failure;
1858
1859                 stats = rcu_dereference_check(basechain->stats,
1860                                               lockdep_commit_lock_is_held(net));
1861                 if (nft_dump_stats(skb, stats))
1862                         goto nla_put_failure;
1863         }
1864
1865         if (chain->flags &&
1866             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1867                 goto nla_put_failure;
1868
1869         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1870                 goto nla_put_failure;
1871
1872         if (chain->udata &&
1873             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1874                 goto nla_put_failure;
1875
1876         nlmsg_end(skb, nlh);
1877         return 0;
1878
1879 nla_put_failure:
1880         nlmsg_trim(skb, nlh);
1881         return -1;
1882 }
1883
1884 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1885                                    const struct list_head *hook_list)
1886 {
1887         struct nftables_pernet *nft_net;
1888         struct sk_buff *skb;
1889         u16 flags = 0;
1890         int err;
1891
1892         if (!ctx->report &&
1893             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1894                 return;
1895
1896         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1897         if (skb == NULL)
1898                 goto err;
1899
1900         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1901                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1902
1903         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1904                                         event, flags, ctx->family, ctx->table,
1905                                         ctx->chain, hook_list);
1906         if (err < 0) {
1907                 kfree_skb(skb);
1908                 goto err;
1909         }
1910
1911         nft_net = nft_pernet(ctx->net);
1912         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1913         return;
1914 err:
1915         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1916 }
1917
1918 static int nf_tables_dump_chains(struct sk_buff *skb,
1919                                  struct netlink_callback *cb)
1920 {
1921         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1922         unsigned int idx = 0, s_idx = cb->args[0];
1923         struct net *net = sock_net(skb->sk);
1924         int family = nfmsg->nfgen_family;
1925         struct nftables_pernet *nft_net;
1926         const struct nft_table *table;
1927         const struct nft_chain *chain;
1928
1929         rcu_read_lock();
1930         nft_net = nft_pernet(net);
1931         cb->seq = READ_ONCE(nft_net->base_seq);
1932
1933         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1934                 if (family != NFPROTO_UNSPEC && family != table->family)
1935                         continue;
1936
1937                 list_for_each_entry_rcu(chain, &table->chains, list) {
1938                         if (idx < s_idx)
1939                                 goto cont;
1940                         if (idx > s_idx)
1941                                 memset(&cb->args[1], 0,
1942                                        sizeof(cb->args) - sizeof(cb->args[0]));
1943                         if (!nft_is_active(net, chain))
1944                                 continue;
1945                         if (nf_tables_fill_chain_info(skb, net,
1946                                                       NETLINK_CB(cb->skb).portid,
1947                                                       cb->nlh->nlmsg_seq,
1948                                                       NFT_MSG_NEWCHAIN,
1949                                                       NLM_F_MULTI,
1950                                                       table->family, table,
1951                                                       chain, NULL) < 0)
1952                                 goto done;
1953
1954                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1955 cont:
1956                         idx++;
1957                 }
1958         }
1959 done:
1960         rcu_read_unlock();
1961         cb->args[0] = idx;
1962         return skb->len;
1963 }
1964
1965 /* called with rcu_read_lock held */
1966 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1967                               const struct nlattr * const nla[])
1968 {
1969         struct netlink_ext_ack *extack = info->extack;
1970         u8 genmask = nft_genmask_cur(info->net);
1971         u8 family = info->nfmsg->nfgen_family;
1972         const struct nft_chain *chain;
1973         struct net *net = info->net;
1974         struct nft_table *table;
1975         struct sk_buff *skb2;
1976         int err;
1977
1978         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1979                 struct netlink_dump_control c = {
1980                         .dump = nf_tables_dump_chains,
1981                         .module = THIS_MODULE,
1982                 };
1983
1984                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1985         }
1986
1987         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1988         if (IS_ERR(table)) {
1989                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1990                 return PTR_ERR(table);
1991         }
1992
1993         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1994         if (IS_ERR(chain)) {
1995                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1996                 return PTR_ERR(chain);
1997         }
1998
1999         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
2000         if (!skb2)
2001                 return -ENOMEM;
2002
2003         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
2004                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
2005                                         0, family, table, chain, NULL);
2006         if (err < 0)
2007                 goto err_fill_chain_info;
2008
2009         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
2010
2011 err_fill_chain_info:
2012         kfree_skb(skb2);
2013         return err;
2014 }
2015
2016 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
2017         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
2018         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
2019 };
2020
2021 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
2022 {
2023         struct nlattr *tb[NFTA_COUNTER_MAX+1];
2024         struct nft_stats __percpu *newstats;
2025         struct nft_stats *stats;
2026         int err;
2027
2028         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
2029                                           nft_counter_policy, NULL);
2030         if (err < 0)
2031                 return ERR_PTR(err);
2032
2033         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
2034                 return ERR_PTR(-EINVAL);
2035
2036         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
2037         if (newstats == NULL)
2038                 return ERR_PTR(-ENOMEM);
2039
2040         /* Restore old counters on this cpu, no problem. Per-cpu statistics
2041          * are not exposed to userspace.
2042          */
2043         preempt_disable();
2044         stats = this_cpu_ptr(newstats);
2045         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
2046         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
2047         preempt_enable();
2048
2049         return newstats;
2050 }
2051
2052 static void nft_chain_stats_replace(struct nft_trans *trans)
2053 {
2054         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
2055
2056         if (!nft_trans_chain_stats(trans))
2057                 return;
2058
2059         nft_trans_chain_stats(trans) =
2060                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
2061                                     lockdep_commit_lock_is_held(trans->ctx.net));
2062
2063         if (!nft_trans_chain_stats(trans))
2064                 static_branch_inc(&nft_counters_enabled);
2065 }
2066
2067 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
2068 {
2069         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
2070         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
2071
2072         if (g0 != g1)
2073                 kvfree(g1);
2074         kvfree(g0);
2075
2076         /* should be NULL either via abort or via successful commit */
2077         WARN_ON_ONCE(chain->blob_next);
2078         kvfree(chain->blob_next);
2079 }
2080
2081 void nf_tables_chain_destroy(struct nft_ctx *ctx)
2082 {
2083         struct nft_chain *chain = ctx->chain;
2084         struct nft_hook *hook, *next;
2085
2086         if (WARN_ON(chain->use > 0))
2087                 return;
2088
2089         /* no concurrent access possible anymore */
2090         nf_tables_chain_free_chain_rules(chain);
2091
2092         if (nft_is_base_chain(chain)) {
2093                 struct nft_base_chain *basechain = nft_base_chain(chain);
2094
2095                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2096                         list_for_each_entry_safe(hook, next,
2097                                                  &basechain->hook_list, list) {
2098                                 list_del_rcu(&hook->list);
2099                                 kfree_rcu(hook, rcu);
2100                         }
2101                 }
2102                 module_put(basechain->type->owner);
2103                 if (rcu_access_pointer(basechain->stats)) {
2104                         static_branch_dec(&nft_counters_enabled);
2105                         free_percpu(rcu_dereference_raw(basechain->stats));
2106                 }
2107                 kfree(chain->name);
2108                 kfree(chain->udata);
2109                 kfree(basechain);
2110         } else {
2111                 kfree(chain->name);
2112                 kfree(chain->udata);
2113                 kfree(chain);
2114         }
2115 }
2116
2117 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
2118                                               const struct nlattr *attr)
2119 {
2120         struct net_device *dev;
2121         char ifname[IFNAMSIZ];
2122         struct nft_hook *hook;
2123         int err;
2124
2125         hook = kzalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2126         if (!hook) {
2127                 err = -ENOMEM;
2128                 goto err_hook_alloc;
2129         }
2130
2131         nla_strscpy(ifname, attr, IFNAMSIZ);
2132         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2133          * indirectly serializing all the other holders of the commit_mutex with
2134          * the rtnl_mutex.
2135          */
2136         dev = __dev_get_by_name(net, ifname);
2137         if (!dev) {
2138                 err = -ENOENT;
2139                 goto err_hook_dev;
2140         }
2141         hook->ops.dev = dev;
2142
2143         return hook;
2144
2145 err_hook_dev:
2146         kfree(hook);
2147 err_hook_alloc:
2148         return ERR_PTR(err);
2149 }
2150
2151 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2152                                            const struct nft_hook *this)
2153 {
2154         struct nft_hook *hook;
2155
2156         list_for_each_entry(hook, hook_list, list) {
2157                 if (this->ops.dev == hook->ops.dev)
2158                         return hook;
2159         }
2160
2161         return NULL;
2162 }
2163
2164 static int nf_tables_parse_netdev_hooks(struct net *net,
2165                                         const struct nlattr *attr,
2166                                         struct list_head *hook_list,
2167                                         struct netlink_ext_ack *extack)
2168 {
2169         struct nft_hook *hook, *next;
2170         const struct nlattr *tmp;
2171         int rem, n = 0, err;
2172
2173         nla_for_each_nested(tmp, attr, rem) {
2174                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2175                         err = -EINVAL;
2176                         goto err_hook;
2177                 }
2178
2179                 hook = nft_netdev_hook_alloc(net, tmp);
2180                 if (IS_ERR(hook)) {
2181                         NL_SET_BAD_ATTR(extack, tmp);
2182                         err = PTR_ERR(hook);
2183                         goto err_hook;
2184                 }
2185                 if (nft_hook_list_find(hook_list, hook)) {
2186                         NL_SET_BAD_ATTR(extack, tmp);
2187                         kfree(hook);
2188                         err = -EEXIST;
2189                         goto err_hook;
2190                 }
2191                 list_add_tail(&hook->list, hook_list);
2192                 n++;
2193
2194                 if (n == NFT_NETDEVICE_MAX) {
2195                         err = -EFBIG;
2196                         goto err_hook;
2197                 }
2198         }
2199
2200         return 0;
2201
2202 err_hook:
2203         list_for_each_entry_safe(hook, next, hook_list, list) {
2204                 list_del(&hook->list);
2205                 kfree(hook);
2206         }
2207         return err;
2208 }
2209
2210 struct nft_chain_hook {
2211         u32                             num;
2212         s32                             priority;
2213         const struct nft_chain_type     *type;
2214         struct list_head                list;
2215 };
2216
2217 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2218                                   struct list_head *hook_list,
2219                                   struct netlink_ext_ack *extack, u32 flags)
2220 {
2221         struct nft_hook *hook;
2222         int err;
2223
2224         if (tb[NFTA_HOOK_DEV]) {
2225                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2226                 if (IS_ERR(hook)) {
2227                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2228                         return PTR_ERR(hook);
2229                 }
2230
2231                 list_add_tail(&hook->list, hook_list);
2232         } else if (tb[NFTA_HOOK_DEVS]) {
2233                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2234                                                    hook_list, extack);
2235                 if (err < 0)
2236                         return err;
2237
2238         }
2239
2240         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2241             list_empty(hook_list))
2242                 return -EINVAL;
2243
2244         return 0;
2245 }
2246
2247 static int nft_chain_parse_hook(struct net *net,
2248                                 struct nft_base_chain *basechain,
2249                                 const struct nlattr * const nla[],
2250                                 struct nft_chain_hook *hook, u8 family,
2251                                 u32 flags, struct netlink_ext_ack *extack)
2252 {
2253         struct nftables_pernet *nft_net = nft_pernet(net);
2254         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2255         const struct nft_chain_type *type;
2256         int err;
2257
2258         lockdep_assert_held(&nft_net->commit_mutex);
2259         lockdep_nfnl_nft_mutex_not_held();
2260
2261         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2262                                           nla[NFTA_CHAIN_HOOK],
2263                                           nft_hook_policy, NULL);
2264         if (err < 0)
2265                 return err;
2266
2267         if (!basechain) {
2268                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2269                     !ha[NFTA_HOOK_PRIORITY]) {
2270                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2271                         return -ENOENT;
2272                 }
2273
2274                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2275                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2276
2277                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2278                 if (!type)
2279                         return -EOPNOTSUPP;
2280
2281                 if (nla[NFTA_CHAIN_TYPE]) {
2282                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2283                                                            family, true);
2284                         if (IS_ERR(type)) {
2285                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2286                                 return PTR_ERR(type);
2287                         }
2288                 }
2289                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2290                         return -EOPNOTSUPP;
2291
2292                 if (type->type == NFT_CHAIN_T_NAT &&
2293                     hook->priority <= NF_IP_PRI_CONNTRACK)
2294                         return -EOPNOTSUPP;
2295         } else {
2296                 if (ha[NFTA_HOOK_HOOKNUM]) {
2297                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2298                         if (hook->num != basechain->ops.hooknum)
2299                                 return -EOPNOTSUPP;
2300                 }
2301                 if (ha[NFTA_HOOK_PRIORITY]) {
2302                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2303                         if (hook->priority != basechain->ops.priority)
2304                                 return -EOPNOTSUPP;
2305                 }
2306
2307                 if (nla[NFTA_CHAIN_TYPE]) {
2308                         type = __nf_tables_chain_type_lookup(nla[NFTA_CHAIN_TYPE],
2309                                                              family);
2310                         if (!type) {
2311                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2312                                 return -ENOENT;
2313                         }
2314                 } else {
2315                         type = basechain->type;
2316                 }
2317         }
2318
2319         if (!try_module_get(type->owner)) {
2320                 if (nla[NFTA_CHAIN_TYPE])
2321                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2322                 return -ENOENT;
2323         }
2324
2325         hook->type = type;
2326
2327         INIT_LIST_HEAD(&hook->list);
2328         if (nft_base_chain_netdev(family, hook->num)) {
2329                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2330                 if (err < 0) {
2331                         module_put(type->owner);
2332                         return err;
2333                 }
2334         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2335                 module_put(type->owner);
2336                 return -EOPNOTSUPP;
2337         }
2338
2339         return 0;
2340 }
2341
2342 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2343 {
2344         struct nft_hook *h, *next;
2345
2346         list_for_each_entry_safe(h, next, &hook->list, list) {
2347                 list_del(&h->list);
2348                 kfree(h);
2349         }
2350         module_put(hook->type->owner);
2351 }
2352
2353 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2354 {
2355         struct nft_rule_dp_last *lrule;
2356
2357         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2358
2359         lrule = (struct nft_rule_dp_last *)ptr;
2360         lrule->end.is_last = 1;
2361         lrule->chain = chain;
2362         /* blob size does not include the trailer rule */
2363 }
2364
2365 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2366                                                          unsigned int size)
2367 {
2368         struct nft_rule_blob *blob;
2369
2370         if (size > INT_MAX)
2371                 return NULL;
2372
2373         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2374
2375         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2376         if (!blob)
2377                 return NULL;
2378
2379         blob->size = 0;
2380         nft_last_rule(chain, blob->data);
2381
2382         return blob;
2383 }
2384
2385 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2386                                     const struct nft_chain_hook *hook,
2387                                     struct nft_chain *chain)
2388 {
2389         ops->pf                 = family;
2390         ops->hooknum            = hook->num;
2391         ops->priority           = hook->priority;
2392         ops->priv               = chain;
2393         ops->hook               = hook->type->hooks[ops->hooknum];
2394         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2395 }
2396
2397 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2398                               struct nft_chain_hook *hook, u32 flags)
2399 {
2400         struct nft_chain *chain;
2401         struct nft_hook *h;
2402
2403         basechain->type = hook->type;
2404         INIT_LIST_HEAD(&basechain->hook_list);
2405         chain = &basechain->chain;
2406
2407         if (nft_base_chain_netdev(family, hook->num)) {
2408                 list_splice_init(&hook->list, &basechain->hook_list);
2409                 list_for_each_entry(h, &basechain->hook_list, list)
2410                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2411         }
2412         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2413
2414         chain->flags |= NFT_CHAIN_BASE | flags;
2415         basechain->policy = NF_ACCEPT;
2416         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2417             !nft_chain_offload_support(basechain)) {
2418                 list_splice_init(&basechain->hook_list, &hook->list);
2419                 return -EOPNOTSUPP;
2420         }
2421
2422         flow_block_init(&basechain->flow_block);
2423
2424         return 0;
2425 }
2426
2427 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2428 {
2429         int err;
2430
2431         err = rhltable_insert_key(&table->chains_ht, chain->name,
2432                                   &chain->rhlhead, nft_chain_ht_params);
2433         if (err)
2434                 return err;
2435
2436         list_add_tail_rcu(&chain->list, &table->chains);
2437
2438         return 0;
2439 }
2440
2441 static u64 chain_id;
2442
2443 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2444                               u8 policy, u32 flags,
2445                               struct netlink_ext_ack *extack)
2446 {
2447         const struct nlattr * const *nla = ctx->nla;
2448         struct nft_table *table = ctx->table;
2449         struct nft_base_chain *basechain;
2450         struct net *net = ctx->net;
2451         char name[NFT_NAME_MAXLEN];
2452         struct nft_rule_blob *blob;
2453         struct nft_trans *trans;
2454         struct nft_chain *chain;
2455         int err;
2456
2457         if (nla[NFTA_CHAIN_HOOK]) {
2458                 struct nft_stats __percpu *stats = NULL;
2459                 struct nft_chain_hook hook = {};
2460
2461                 if (table->flags & __NFT_TABLE_F_UPDATE)
2462                         return -EINVAL;
2463
2464                 if (flags & NFT_CHAIN_BINDING)
2465                         return -EOPNOTSUPP;
2466
2467                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2468                                            extack);
2469                 if (err < 0)
2470                         return err;
2471
2472                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2473                 if (basechain == NULL) {
2474                         nft_chain_release_hook(&hook);
2475                         return -ENOMEM;
2476                 }
2477                 chain = &basechain->chain;
2478
2479                 if (nla[NFTA_CHAIN_COUNTERS]) {
2480                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2481                         if (IS_ERR(stats)) {
2482                                 nft_chain_release_hook(&hook);
2483                                 kfree(basechain);
2484                                 return PTR_ERR(stats);
2485                         }
2486                         rcu_assign_pointer(basechain->stats, stats);
2487                 }
2488
2489                 err = nft_basechain_init(basechain, family, &hook, flags);
2490                 if (err < 0) {
2491                         nft_chain_release_hook(&hook);
2492                         kfree(basechain);
2493                         free_percpu(stats);
2494                         return err;
2495                 }
2496                 if (stats)
2497                         static_branch_inc(&nft_counters_enabled);
2498         } else {
2499                 if (flags & NFT_CHAIN_BASE)
2500                         return -EINVAL;
2501                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2502                         return -EOPNOTSUPP;
2503
2504                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2505                 if (chain == NULL)
2506                         return -ENOMEM;
2507
2508                 chain->flags = flags;
2509         }
2510         ctx->chain = chain;
2511
2512         INIT_LIST_HEAD(&chain->rules);
2513         chain->handle = nf_tables_alloc_handle(table);
2514         chain->table = table;
2515
2516         if (nla[NFTA_CHAIN_NAME]) {
2517                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2518         } else {
2519                 if (!(flags & NFT_CHAIN_BINDING)) {
2520                         err = -EINVAL;
2521                         goto err_destroy_chain;
2522                 }
2523
2524                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2525                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2526         }
2527
2528         if (!chain->name) {
2529                 err = -ENOMEM;
2530                 goto err_destroy_chain;
2531         }
2532
2533         if (nla[NFTA_CHAIN_USERDATA]) {
2534                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2535                 if (chain->udata == NULL) {
2536                         err = -ENOMEM;
2537                         goto err_destroy_chain;
2538                 }
2539                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2540         }
2541
2542         blob = nf_tables_chain_alloc_rules(chain, 0);
2543         if (!blob) {
2544                 err = -ENOMEM;
2545                 goto err_destroy_chain;
2546         }
2547
2548         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2549         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2550
2551         if (!nft_use_inc(&table->use)) {
2552                 err = -EMFILE;
2553                 goto err_destroy_chain;
2554         }
2555
2556         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2557         if (IS_ERR(trans)) {
2558                 err = PTR_ERR(trans);
2559                 goto err_trans;
2560         }
2561
2562         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2563         if (nft_is_base_chain(chain))
2564                 nft_trans_chain_policy(trans) = policy;
2565
2566         err = nft_chain_add(table, chain);
2567         if (err < 0)
2568                 goto err_chain_add;
2569
2570         /* This must be LAST to ensure no packets are walking over this chain. */
2571         err = nf_tables_register_hook(net, table, chain);
2572         if (err < 0)
2573                 goto err_register_hook;
2574
2575         return 0;
2576
2577 err_register_hook:
2578         nft_chain_del(chain);
2579 err_chain_add:
2580         nft_trans_destroy(trans);
2581 err_trans:
2582         nft_use_dec_restore(&table->use);
2583 err_destroy_chain:
2584         nf_tables_chain_destroy(ctx);
2585
2586         return err;
2587 }
2588
2589 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2590                               u32 flags, const struct nlattr *attr,
2591                               struct netlink_ext_ack *extack)
2592 {
2593         const struct nlattr * const *nla = ctx->nla;
2594         struct nft_base_chain *basechain = NULL;
2595         struct nft_table *table = ctx->table;
2596         struct nft_chain *chain = ctx->chain;
2597         struct nft_chain_hook hook = {};
2598         struct nft_stats *stats = NULL;
2599         struct nft_hook *h, *next;
2600         struct nf_hook_ops *ops;
2601         struct nft_trans *trans;
2602         bool unregister = false;
2603         int err;
2604
2605         if (chain->flags ^ flags)
2606                 return -EOPNOTSUPP;
2607
2608         INIT_LIST_HEAD(&hook.list);
2609
2610         if (nla[NFTA_CHAIN_HOOK]) {
2611                 if (!nft_is_base_chain(chain)) {
2612                         NL_SET_BAD_ATTR(extack, attr);
2613                         return -EEXIST;
2614                 }
2615
2616                 basechain = nft_base_chain(chain);
2617                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2618                                            ctx->family, flags, extack);
2619                 if (err < 0)
2620                         return err;
2621
2622                 if (basechain->type != hook.type) {
2623                         nft_chain_release_hook(&hook);
2624                         NL_SET_BAD_ATTR(extack, attr);
2625                         return -EEXIST;
2626                 }
2627
2628                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2629                         list_for_each_entry_safe(h, next, &hook.list, list) {
2630                                 h->ops.pf       = basechain->ops.pf;
2631                                 h->ops.hooknum  = basechain->ops.hooknum;
2632                                 h->ops.priority = basechain->ops.priority;
2633                                 h->ops.priv     = basechain->ops.priv;
2634                                 h->ops.hook     = basechain->ops.hook;
2635
2636                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2637                                         list_del(&h->list);
2638                                         kfree(h);
2639                                 }
2640                         }
2641                 } else {
2642                         ops = &basechain->ops;
2643                         if (ops->hooknum != hook.num ||
2644                             ops->priority != hook.priority) {
2645                                 nft_chain_release_hook(&hook);
2646                                 NL_SET_BAD_ATTR(extack, attr);
2647                                 return -EEXIST;
2648                         }
2649                 }
2650         }
2651
2652         if (nla[NFTA_CHAIN_HANDLE] &&
2653             nla[NFTA_CHAIN_NAME]) {
2654                 struct nft_chain *chain2;
2655
2656                 chain2 = nft_chain_lookup(ctx->net, table,
2657                                           nla[NFTA_CHAIN_NAME], genmask);
2658                 if (!IS_ERR(chain2)) {
2659                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2660                         err = -EEXIST;
2661                         goto err_hooks;
2662                 }
2663         }
2664
2665         if (table->flags & __NFT_TABLE_F_UPDATE &&
2666             !list_empty(&hook.list)) {
2667                 NL_SET_BAD_ATTR(extack, attr);
2668                 err = -EOPNOTSUPP;
2669                 goto err_hooks;
2670         }
2671
2672         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2673             nft_is_base_chain(chain) &&
2674             !list_empty(&hook.list)) {
2675                 basechain = nft_base_chain(chain);
2676                 ops = &basechain->ops;
2677
2678                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2679                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2680                         if (err < 0)
2681                                 goto err_hooks;
2682                 }
2683         }
2684
2685         unregister = true;
2686
2687         if (nla[NFTA_CHAIN_COUNTERS]) {
2688                 if (!nft_is_base_chain(chain)) {
2689                         err = -EOPNOTSUPP;
2690                         goto err_hooks;
2691                 }
2692
2693                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2694                 if (IS_ERR(stats)) {
2695                         err = PTR_ERR(stats);
2696                         goto err_hooks;
2697                 }
2698         }
2699
2700         err = -ENOMEM;
2701         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2702                                 sizeof(struct nft_trans_chain));
2703         if (trans == NULL)
2704                 goto err_trans;
2705
2706         nft_trans_chain_stats(trans) = stats;
2707         nft_trans_chain_update(trans) = true;
2708
2709         if (nla[NFTA_CHAIN_POLICY])
2710                 nft_trans_chain_policy(trans) = policy;
2711         else
2712                 nft_trans_chain_policy(trans) = -1;
2713
2714         if (nla[NFTA_CHAIN_HANDLE] &&
2715             nla[NFTA_CHAIN_NAME]) {
2716                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2717                 struct nft_trans *tmp;
2718                 char *name;
2719
2720                 err = -ENOMEM;
2721                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2722                 if (!name)
2723                         goto err_trans;
2724
2725                 err = -EEXIST;
2726                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2727                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2728                             tmp->ctx.table == table &&
2729                             nft_trans_chain_update(tmp) &&
2730                             nft_trans_chain_name(tmp) &&
2731                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2732                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2733                                 kfree(name);
2734                                 goto err_trans;
2735                         }
2736                 }
2737
2738                 nft_trans_chain_name(trans) = name;
2739         }
2740
2741         nft_trans_basechain(trans) = basechain;
2742         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2743         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2744         if (nla[NFTA_CHAIN_HOOK])
2745                 module_put(hook.type->owner);
2746
2747         nft_trans_commit_list_add_tail(ctx->net, trans);
2748
2749         return 0;
2750
2751 err_trans:
2752         free_percpu(stats);
2753         kfree(trans);
2754 err_hooks:
2755         if (nla[NFTA_CHAIN_HOOK]) {
2756                 list_for_each_entry_safe(h, next, &hook.list, list) {
2757                         if (unregister)
2758                                 nf_unregister_net_hook(ctx->net, &h->ops);
2759                         list_del(&h->list);
2760                         kfree_rcu(h, rcu);
2761                 }
2762                 module_put(hook.type->owner);
2763         }
2764
2765         return err;
2766 }
2767
2768 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2769                                                const struct nft_table *table,
2770                                                const struct nlattr *nla, u8 genmask)
2771 {
2772         struct nftables_pernet *nft_net = nft_pernet(net);
2773         u32 id = ntohl(nla_get_be32(nla));
2774         struct nft_trans *trans;
2775
2776         list_for_each_entry(trans, &nft_net->commit_list, list) {
2777                 struct nft_chain *chain = trans->ctx.chain;
2778
2779                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2780                     chain->table == table &&
2781                     id == nft_trans_chain_id(trans) &&
2782                     nft_active_genmask(chain, genmask))
2783                         return chain;
2784         }
2785         return ERR_PTR(-ENOENT);
2786 }
2787
2788 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2789                               const struct nlattr * const nla[])
2790 {
2791         struct nftables_pernet *nft_net = nft_pernet(info->net);
2792         struct netlink_ext_ack *extack = info->extack;
2793         u8 genmask = nft_genmask_next(info->net);
2794         u8 family = info->nfmsg->nfgen_family;
2795         struct nft_chain *chain = NULL;
2796         struct net *net = info->net;
2797         const struct nlattr *attr;
2798         struct nft_table *table;
2799         u8 policy = NF_ACCEPT;
2800         struct nft_ctx ctx;
2801         u64 handle = 0;
2802         u32 flags = 0;
2803
2804         lockdep_assert_held(&nft_net->commit_mutex);
2805
2806         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2807                                  NETLINK_CB(skb).portid);
2808         if (IS_ERR(table)) {
2809                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2810                 return PTR_ERR(table);
2811         }
2812
2813         chain = NULL;
2814         attr = nla[NFTA_CHAIN_NAME];
2815
2816         if (nla[NFTA_CHAIN_HANDLE]) {
2817                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2818                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2819                 if (IS_ERR(chain)) {
2820                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2821                         return PTR_ERR(chain);
2822                 }
2823                 attr = nla[NFTA_CHAIN_HANDLE];
2824         } else if (nla[NFTA_CHAIN_NAME]) {
2825                 chain = nft_chain_lookup(net, table, attr, genmask);
2826                 if (IS_ERR(chain)) {
2827                         if (PTR_ERR(chain) != -ENOENT) {
2828                                 NL_SET_BAD_ATTR(extack, attr);
2829                                 return PTR_ERR(chain);
2830                         }
2831                         chain = NULL;
2832                 }
2833         } else if (!nla[NFTA_CHAIN_ID]) {
2834                 return -EINVAL;
2835         }
2836
2837         if (nla[NFTA_CHAIN_POLICY]) {
2838                 if (chain != NULL &&
2839                     !nft_is_base_chain(chain)) {
2840                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2841                         return -EOPNOTSUPP;
2842                 }
2843
2844                 if (chain == NULL &&
2845                     nla[NFTA_CHAIN_HOOK] == NULL) {
2846                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2847                         return -EOPNOTSUPP;
2848                 }
2849
2850                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2851                 switch (policy) {
2852                 case NF_DROP:
2853                 case NF_ACCEPT:
2854                         break;
2855                 default:
2856                         return -EINVAL;
2857                 }
2858         }
2859
2860         if (nla[NFTA_CHAIN_FLAGS])
2861                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2862         else if (chain)
2863                 flags = chain->flags;
2864
2865         if (flags & ~NFT_CHAIN_FLAGS)
2866                 return -EOPNOTSUPP;
2867
2868         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2869
2870         if (chain != NULL) {
2871                 if (chain->flags & NFT_CHAIN_BINDING)
2872                         return -EINVAL;
2873
2874                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2875                         NL_SET_BAD_ATTR(extack, attr);
2876                         return -EEXIST;
2877                 }
2878                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2879                         return -EOPNOTSUPP;
2880
2881                 flags |= chain->flags & NFT_CHAIN_BASE;
2882                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2883                                           extack);
2884         }
2885
2886         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2887 }
2888
2889 static int nft_delchain_hook(struct nft_ctx *ctx,
2890                              struct nft_base_chain *basechain,
2891                              struct netlink_ext_ack *extack)
2892 {
2893         const struct nft_chain *chain = &basechain->chain;
2894         const struct nlattr * const *nla = ctx->nla;
2895         struct nft_chain_hook chain_hook = {};
2896         struct nft_hook *this, *hook;
2897         LIST_HEAD(chain_del_list);
2898         struct nft_trans *trans;
2899         int err;
2900
2901         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
2902                 return -EOPNOTSUPP;
2903
2904         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2905                                    ctx->family, chain->flags, extack);
2906         if (err < 0)
2907                 return err;
2908
2909         list_for_each_entry(this, &chain_hook.list, list) {
2910                 hook = nft_hook_list_find(&basechain->hook_list, this);
2911                 if (!hook) {
2912                         err = -ENOENT;
2913                         goto err_chain_del_hook;
2914                 }
2915                 list_move(&hook->list, &chain_del_list);
2916         }
2917
2918         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2919                                 sizeof(struct nft_trans_chain));
2920         if (!trans) {
2921                 err = -ENOMEM;
2922                 goto err_chain_del_hook;
2923         }
2924
2925         nft_trans_basechain(trans) = basechain;
2926         nft_trans_chain_update(trans) = true;
2927         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2928         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2929         nft_chain_release_hook(&chain_hook);
2930
2931         nft_trans_commit_list_add_tail(ctx->net, trans);
2932
2933         return 0;
2934
2935 err_chain_del_hook:
2936         list_splice(&chain_del_list, &basechain->hook_list);
2937         nft_chain_release_hook(&chain_hook);
2938
2939         return err;
2940 }
2941
2942 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2943                               const struct nlattr * const nla[])
2944 {
2945         struct netlink_ext_ack *extack = info->extack;
2946         u8 genmask = nft_genmask_next(info->net);
2947         u8 family = info->nfmsg->nfgen_family;
2948         struct net *net = info->net;
2949         const struct nlattr *attr;
2950         struct nft_table *table;
2951         struct nft_chain *chain;
2952         struct nft_rule *rule;
2953         struct nft_ctx ctx;
2954         u64 handle;
2955         u32 use;
2956         int err;
2957
2958         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2959                                  NETLINK_CB(skb).portid);
2960         if (IS_ERR(table)) {
2961                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2962                 return PTR_ERR(table);
2963         }
2964
2965         if (nla[NFTA_CHAIN_HANDLE]) {
2966                 attr = nla[NFTA_CHAIN_HANDLE];
2967                 handle = be64_to_cpu(nla_get_be64(attr));
2968                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2969         } else {
2970                 attr = nla[NFTA_CHAIN_NAME];
2971                 chain = nft_chain_lookup(net, table, attr, genmask);
2972         }
2973         if (IS_ERR(chain)) {
2974                 if (PTR_ERR(chain) == -ENOENT &&
2975                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2976                         return 0;
2977
2978                 NL_SET_BAD_ATTR(extack, attr);
2979                 return PTR_ERR(chain);
2980         }
2981
2982         if (nft_chain_binding(chain))
2983                 return -EOPNOTSUPP;
2984
2985         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2986
2987         if (nla[NFTA_CHAIN_HOOK]) {
2988                 if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN ||
2989                     chain->flags & NFT_CHAIN_HW_OFFLOAD)
2990                         return -EOPNOTSUPP;
2991
2992                 if (nft_is_base_chain(chain)) {
2993                         struct nft_base_chain *basechain = nft_base_chain(chain);
2994
2995                         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
2996                                 return nft_delchain_hook(&ctx, basechain, extack);
2997                 }
2998         }
2999
3000         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
3001             chain->use > 0)
3002                 return -EBUSY;
3003
3004         use = chain->use;
3005         list_for_each_entry(rule, &chain->rules, list) {
3006                 if (!nft_is_active_next(net, rule))
3007                         continue;
3008                 use--;
3009
3010                 err = nft_delrule(&ctx, rule);
3011                 if (err < 0)
3012                         return err;
3013         }
3014
3015         /* There are rules and elements that are still holding references to us,
3016          * we cannot do a recursive removal in this case.
3017          */
3018         if (use > 0) {
3019                 NL_SET_BAD_ATTR(extack, attr);
3020                 return -EBUSY;
3021         }
3022
3023         return nft_delchain(&ctx);
3024 }
3025
3026 /*
3027  * Expressions
3028  */
3029
3030 /**
3031  *      nft_register_expr - register nf_tables expr type
3032  *      @type: expr type
3033  *
3034  *      Registers the expr type for use with nf_tables. Returns zero on
3035  *      success or a negative errno code otherwise.
3036  */
3037 int nft_register_expr(struct nft_expr_type *type)
3038 {
3039         if (WARN_ON_ONCE(type->maxattr > NFT_EXPR_MAXATTR))
3040                 return -ENOMEM;
3041
3042         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3043         if (type->family == NFPROTO_UNSPEC)
3044                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
3045         else
3046                 list_add_rcu(&type->list, &nf_tables_expressions);
3047         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3048         return 0;
3049 }
3050 EXPORT_SYMBOL_GPL(nft_register_expr);
3051
3052 /**
3053  *      nft_unregister_expr - unregister nf_tables expr type
3054  *      @type: expr type
3055  *
3056  *      Unregisters the expr typefor use with nf_tables.
3057  */
3058 void nft_unregister_expr(struct nft_expr_type *type)
3059 {
3060         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3061         list_del_rcu(&type->list);
3062         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3063 }
3064 EXPORT_SYMBOL_GPL(nft_unregister_expr);
3065
3066 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
3067                                                        struct nlattr *nla)
3068 {
3069         const struct nft_expr_type *type, *candidate = NULL;
3070
3071         list_for_each_entry_rcu(type, &nf_tables_expressions, list) {
3072                 if (!nla_strcmp(nla, type->name)) {
3073                         if (!type->family && !candidate)
3074                                 candidate = type;
3075                         else if (type->family == family)
3076                                 candidate = type;
3077                 }
3078         }
3079         return candidate;
3080 }
3081
3082 #ifdef CONFIG_MODULES
3083 static int nft_expr_type_request_module(struct net *net, u8 family,
3084                                         struct nlattr *nla)
3085 {
3086         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
3087                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
3088                 return -EAGAIN;
3089
3090         return 0;
3091 }
3092 #endif
3093
3094 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
3095                                                      u8 family,
3096                                                      struct nlattr *nla)
3097 {
3098         const struct nft_expr_type *type;
3099
3100         if (nla == NULL)
3101                 return ERR_PTR(-EINVAL);
3102
3103         rcu_read_lock();
3104         type = __nft_expr_type_get(family, nla);
3105         if (type != NULL && try_module_get(type->owner)) {
3106                 rcu_read_unlock();
3107                 return type;
3108         }
3109         rcu_read_unlock();
3110
3111         lockdep_nfnl_nft_mutex_not_held();
3112 #ifdef CONFIG_MODULES
3113         if (type == NULL) {
3114                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
3115                         return ERR_PTR(-EAGAIN);
3116
3117                 if (nft_request_module(net, "nft-expr-%.*s",
3118                                        nla_len(nla),
3119                                        (char *)nla_data(nla)) == -EAGAIN)
3120                         return ERR_PTR(-EAGAIN);
3121         }
3122 #endif
3123         return ERR_PTR(-ENOENT);
3124 }
3125
3126 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
3127         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
3128                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
3129         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
3130 };
3131
3132 static int nf_tables_fill_expr_info(struct sk_buff *skb,
3133                                     const struct nft_expr *expr, bool reset)
3134 {
3135         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
3136                 goto nla_put_failure;
3137
3138         if (expr->ops->dump) {
3139                 struct nlattr *data = nla_nest_start_noflag(skb,
3140                                                             NFTA_EXPR_DATA);
3141                 if (data == NULL)
3142                         goto nla_put_failure;
3143                 if (expr->ops->dump(skb, expr, reset) < 0)
3144                         goto nla_put_failure;
3145                 nla_nest_end(skb, data);
3146         }
3147
3148         return skb->len;
3149
3150 nla_put_failure:
3151         return -1;
3152 };
3153
3154 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
3155                   const struct nft_expr *expr, bool reset)
3156 {
3157         struct nlattr *nest;
3158
3159         nest = nla_nest_start_noflag(skb, attr);
3160         if (!nest)
3161                 goto nla_put_failure;
3162         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
3163                 goto nla_put_failure;
3164         nla_nest_end(skb, nest);
3165         return 0;
3166
3167 nla_put_failure:
3168         return -1;
3169 }
3170
3171 struct nft_expr_info {
3172         const struct nft_expr_ops       *ops;
3173         const struct nlattr             *attr;
3174         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3175 };
3176
3177 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3178                                 const struct nlattr *nla,
3179                                 struct nft_expr_info *info)
3180 {
3181         const struct nft_expr_type *type;
3182         const struct nft_expr_ops *ops;
3183         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3184         int err;
3185
3186         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3187                                           nft_expr_policy, NULL);
3188         if (err < 0)
3189                 return err;
3190
3191         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3192         if (IS_ERR(type))
3193                 return PTR_ERR(type);
3194
3195         if (tb[NFTA_EXPR_DATA]) {
3196                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3197                                                   tb[NFTA_EXPR_DATA],
3198                                                   type->policy, NULL);
3199                 if (err < 0)
3200                         goto err1;
3201         } else
3202                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3203
3204         if (type->select_ops != NULL) {
3205                 ops = type->select_ops(ctx,
3206                                        (const struct nlattr * const *)info->tb);
3207                 if (IS_ERR(ops)) {
3208                         err = PTR_ERR(ops);
3209 #ifdef CONFIG_MODULES
3210                         if (err == -EAGAIN)
3211                                 if (nft_expr_type_request_module(ctx->net,
3212                                                                  ctx->family,
3213                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3214                                         err = -ENOENT;
3215 #endif
3216                         goto err1;
3217                 }
3218         } else
3219                 ops = type->ops;
3220
3221         info->attr = nla;
3222         info->ops = ops;
3223
3224         return 0;
3225
3226 err1:
3227         module_put(type->owner);
3228         return err;
3229 }
3230
3231 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3232                          struct nft_expr_info *info)
3233 {
3234         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3235         const struct nft_expr_type *type;
3236         int err;
3237
3238         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3239                                           nft_expr_policy, NULL);
3240         if (err < 0)
3241                 return err;
3242
3243         if (!tb[NFTA_EXPR_DATA] || !tb[NFTA_EXPR_NAME])
3244                 return -EINVAL;
3245
3246         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3247         if (!type)
3248                 return -ENOENT;
3249
3250         if (!type->inner_ops)
3251                 return -EOPNOTSUPP;
3252
3253         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3254                                           tb[NFTA_EXPR_DATA],
3255                                           type->policy, NULL);
3256         if (err < 0)
3257                 goto err_nla_parse;
3258
3259         info->attr = nla;
3260         info->ops = type->inner_ops;
3261
3262         return 0;
3263
3264 err_nla_parse:
3265         return err;
3266 }
3267
3268 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3269                              const struct nft_expr_info *expr_info,
3270                              struct nft_expr *expr)
3271 {
3272         const struct nft_expr_ops *ops = expr_info->ops;
3273         int err;
3274
3275         expr->ops = ops;
3276         if (ops->init) {
3277                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3278                 if (err < 0)
3279                         goto err1;
3280         }
3281
3282         return 0;
3283 err1:
3284         expr->ops = NULL;
3285         return err;
3286 }
3287
3288 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3289                                    struct nft_expr *expr)
3290 {
3291         const struct nft_expr_type *type = expr->ops->type;
3292
3293         if (expr->ops->destroy)
3294                 expr->ops->destroy(ctx, expr);
3295         module_put(type->owner);
3296 }
3297
3298 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3299                                       const struct nlattr *nla)
3300 {
3301         struct nft_expr_info expr_info;
3302         struct nft_expr *expr;
3303         struct module *owner;
3304         int err;
3305
3306         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3307         if (err < 0)
3308                 goto err_expr_parse;
3309
3310         err = -EOPNOTSUPP;
3311         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3312                 goto err_expr_stateful;
3313
3314         err = -ENOMEM;
3315         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3316         if (expr == NULL)
3317                 goto err_expr_stateful;
3318
3319         err = nf_tables_newexpr(ctx, &expr_info, expr);
3320         if (err < 0)
3321                 goto err_expr_new;
3322
3323         return expr;
3324 err_expr_new:
3325         kfree(expr);
3326 err_expr_stateful:
3327         owner = expr_info.ops->type->owner;
3328         if (expr_info.ops->type->release_ops)
3329                 expr_info.ops->type->release_ops(expr_info.ops);
3330
3331         module_put(owner);
3332 err_expr_parse:
3333         return ERR_PTR(err);
3334 }
3335
3336 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
3337 {
3338         int err;
3339
3340         if (WARN_ON_ONCE(!src->ops->clone))
3341                 return -EINVAL;
3342
3343         dst->ops = src->ops;
3344         err = src->ops->clone(dst, src);
3345         if (err < 0)
3346                 return err;
3347
3348         __module_get(src->ops->type->owner);
3349
3350         return 0;
3351 }
3352
3353 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3354 {
3355         nf_tables_expr_destroy(ctx, expr);
3356         kfree(expr);
3357 }
3358
3359 /*
3360  * Rules
3361  */
3362
3363 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3364                                           u64 handle)
3365 {
3366         struct nft_rule *rule;
3367
3368         // FIXME: this sucks
3369         list_for_each_entry_rcu(rule, &chain->rules, list) {
3370                 if (handle == rule->handle)
3371                         return rule;
3372         }
3373
3374         return ERR_PTR(-ENOENT);
3375 }
3376
3377 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3378                                         const struct nlattr *nla)
3379 {
3380         if (nla == NULL)
3381                 return ERR_PTR(-EINVAL);
3382
3383         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3384 }
3385
3386 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3387         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3388                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3389         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3390                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3391         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3392         [NFTA_RULE_EXPRESSIONS] = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
3393         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3394         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3395         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3396                                     .len = NFT_USERDATA_MAXLEN },
3397         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3398         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3399         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3400 };
3401
3402 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3403                                     u32 portid, u32 seq, int event,
3404                                     u32 flags, int family,
3405                                     const struct nft_table *table,
3406                                     const struct nft_chain *chain,
3407                                     const struct nft_rule *rule, u64 handle,
3408                                     bool reset)
3409 {
3410         struct nlmsghdr *nlh;
3411         const struct nft_expr *expr, *next;
3412         struct nlattr *list;
3413         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3414
3415         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3416                            nft_base_seq(net));
3417         if (!nlh)
3418                 goto nla_put_failure;
3419
3420         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3421                 goto nla_put_failure;
3422         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3423                 goto nla_put_failure;
3424         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3425                          NFTA_RULE_PAD))
3426                 goto nla_put_failure;
3427
3428         if (event != NFT_MSG_DELRULE && handle) {
3429                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3430                                  NFTA_RULE_PAD))
3431                         goto nla_put_failure;
3432         }
3433
3434         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3435                 nft_flow_rule_stats(chain, rule);
3436
3437         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3438         if (list == NULL)
3439                 goto nla_put_failure;
3440         nft_rule_for_each_expr(expr, next, rule) {
3441                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3442                         goto nla_put_failure;
3443         }
3444         nla_nest_end(skb, list);
3445
3446         if (rule->udata) {
3447                 struct nft_userdata *udata = nft_userdata(rule);
3448                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3449                             udata->data) < 0)
3450                         goto nla_put_failure;
3451         }
3452
3453         nlmsg_end(skb, nlh);
3454         return 0;
3455
3456 nla_put_failure:
3457         nlmsg_trim(skb, nlh);
3458         return -1;
3459 }
3460
3461 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3462                                   const struct nft_rule *rule, int event)
3463 {
3464         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3465         const struct nft_rule *prule;
3466         struct sk_buff *skb;
3467         u64 handle = 0;
3468         u16 flags = 0;
3469         int err;
3470
3471         if (!ctx->report &&
3472             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3473                 return;
3474
3475         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3476         if (skb == NULL)
3477                 goto err;
3478
3479         if (event == NFT_MSG_NEWRULE &&
3480             !list_is_first(&rule->list, &ctx->chain->rules) &&
3481             !list_is_last(&rule->list, &ctx->chain->rules)) {
3482                 prule = list_prev_entry(rule, list);
3483                 handle = prule->handle;
3484         }
3485         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3486                 flags |= NLM_F_APPEND;
3487         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3488                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3489
3490         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3491                                        event, flags, ctx->family, ctx->table,
3492                                        ctx->chain, rule, handle, false);
3493         if (err < 0) {
3494                 kfree_skb(skb);
3495                 goto err;
3496         }
3497
3498         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3499         return;
3500 err:
3501         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3502 }
3503
3504 static void audit_log_rule_reset(const struct nft_table *table,
3505                                  unsigned int base_seq,
3506                                  unsigned int nentries)
3507 {
3508         char *buf = kasprintf(GFP_ATOMIC, "%s:%u",
3509                               table->name, base_seq);
3510
3511         audit_log_nfcfg(buf, table->family, nentries,
3512                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3513         kfree(buf);
3514 }
3515
3516 struct nft_rule_dump_ctx {
3517         unsigned int s_idx;
3518         char *table;
3519         char *chain;
3520         bool reset;
3521 };
3522
3523 static int __nf_tables_dump_rules(struct sk_buff *skb,
3524                                   unsigned int *idx,
3525                                   struct netlink_callback *cb,
3526                                   const struct nft_table *table,
3527                                   const struct nft_chain *chain)
3528 {
3529         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3530         struct net *net = sock_net(skb->sk);
3531         const struct nft_rule *rule, *prule;
3532         unsigned int entries = 0;
3533         int ret = 0;
3534         u64 handle;
3535
3536         prule = NULL;
3537         list_for_each_entry_rcu(rule, &chain->rules, list) {
3538                 if (!nft_is_active(net, rule))
3539                         goto cont_skip;
3540                 if (*idx < ctx->s_idx)
3541                         goto cont;
3542                 if (prule)
3543                         handle = prule->handle;
3544                 else
3545                         handle = 0;
3546
3547                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3548                                         cb->nlh->nlmsg_seq,
3549                                         NFT_MSG_NEWRULE,
3550                                         NLM_F_MULTI | NLM_F_APPEND,
3551                                         table->family,
3552                                         table, chain, rule, handle, ctx->reset) < 0) {
3553                         ret = 1;
3554                         break;
3555                 }
3556                 entries++;
3557                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3558 cont:
3559                 prule = rule;
3560 cont_skip:
3561                 (*idx)++;
3562         }
3563
3564         if (ctx->reset && entries)
3565                 audit_log_rule_reset(table, cb->seq, entries);
3566
3567         return ret;
3568 }
3569
3570 static int nf_tables_dump_rules(struct sk_buff *skb,
3571                                 struct netlink_callback *cb)
3572 {
3573         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3574         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3575         struct nft_table *table;
3576         const struct nft_chain *chain;
3577         unsigned int idx = 0;
3578         struct net *net = sock_net(skb->sk);
3579         int family = nfmsg->nfgen_family;
3580         struct nftables_pernet *nft_net;
3581
3582         rcu_read_lock();
3583         nft_net = nft_pernet(net);
3584         cb->seq = READ_ONCE(nft_net->base_seq);
3585
3586         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3587                 if (family != NFPROTO_UNSPEC && family != table->family)
3588                         continue;
3589
3590                 if (ctx->table && strcmp(ctx->table, table->name) != 0)
3591                         continue;
3592
3593                 if (ctx->table && ctx->chain) {
3594                         struct rhlist_head *list, *tmp;
3595
3596                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3597                                                nft_chain_ht_params);
3598                         if (!list)
3599                                 goto done;
3600
3601                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3602                                 if (!nft_is_active(net, chain))
3603                                         continue;
3604                                 __nf_tables_dump_rules(skb, &idx,
3605                                                        cb, table, chain);
3606                                 break;
3607                         }
3608                         goto done;
3609                 }
3610
3611                 list_for_each_entry_rcu(chain, &table->chains, list) {
3612                         if (__nf_tables_dump_rules(skb, &idx,
3613                                                    cb, table, chain))
3614                                 goto done;
3615                 }
3616
3617                 if (ctx->table)
3618                         break;
3619         }
3620 done:
3621         rcu_read_unlock();
3622
3623         ctx->s_idx = idx;
3624         return skb->len;
3625 }
3626
3627 static int nf_tables_dumpreset_rules(struct sk_buff *skb,
3628                                      struct netlink_callback *cb)
3629 {
3630         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
3631         int ret;
3632
3633         /* Mutex is held is to prevent that two concurrent dump-and-reset calls
3634          * do not underrun counters and quotas. The commit_mutex is used for
3635          * the lack a better lock, this is not transaction path.
3636          */
3637         mutex_lock(&nft_net->commit_mutex);
3638         ret = nf_tables_dump_rules(skb, cb);
3639         mutex_unlock(&nft_net->commit_mutex);
3640
3641         return ret;
3642 }
3643
3644 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3645 {
3646         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3647         const struct nlattr * const *nla = cb->data;
3648
3649         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
3650
3651         if (nla[NFTA_RULE_TABLE]) {
3652                 ctx->table = nla_strdup(nla[NFTA_RULE_TABLE], GFP_ATOMIC);
3653                 if (!ctx->table)
3654                         return -ENOMEM;
3655         }
3656         if (nla[NFTA_RULE_CHAIN]) {
3657                 ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN], GFP_ATOMIC);
3658                 if (!ctx->chain) {
3659                         kfree(ctx->table);
3660                         return -ENOMEM;
3661                 }
3662         }
3663         return 0;
3664 }
3665
3666 static int nf_tables_dumpreset_rules_start(struct netlink_callback *cb)
3667 {
3668         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3669
3670         ctx->reset = true;
3671
3672         return nf_tables_dump_rules_start(cb);
3673 }
3674
3675 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3676 {
3677         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3678
3679         kfree(ctx->table);
3680         kfree(ctx->chain);
3681         return 0;
3682 }
3683
3684 /* called with rcu_read_lock held */
3685 static struct sk_buff *
3686 nf_tables_getrule_single(u32 portid, const struct nfnl_info *info,
3687                          const struct nlattr * const nla[], bool reset)
3688 {
3689         struct netlink_ext_ack *extack = info->extack;
3690         u8 genmask = nft_genmask_cur(info->net);
3691         u8 family = info->nfmsg->nfgen_family;
3692         const struct nft_chain *chain;
3693         const struct nft_rule *rule;
3694         struct net *net = info->net;
3695         struct nft_table *table;
3696         struct sk_buff *skb2;
3697         int err;
3698
3699         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3700         if (IS_ERR(table)) {
3701                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3702                 return ERR_CAST(table);
3703         }
3704
3705         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3706         if (IS_ERR(chain)) {
3707                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3708                 return ERR_CAST(chain);
3709         }
3710
3711         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3712         if (IS_ERR(rule)) {
3713                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3714                 return ERR_CAST(rule);
3715         }
3716
3717         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3718         if (!skb2)
3719                 return ERR_PTR(-ENOMEM);
3720
3721         err = nf_tables_fill_rule_info(skb2, net, portid,
3722                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3723                                        family, table, chain, rule, 0, reset);
3724         if (err < 0) {
3725                 kfree_skb(skb2);
3726                 return ERR_PTR(err);
3727         }
3728
3729         return skb2;
3730 }
3731
3732 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3733                              const struct nlattr * const nla[])
3734 {
3735         u32 portid = NETLINK_CB(skb).portid;
3736         struct net *net = info->net;
3737         struct sk_buff *skb2;
3738
3739         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3740                 struct netlink_dump_control c = {
3741                         .start= nf_tables_dump_rules_start,
3742                         .dump = nf_tables_dump_rules,
3743                         .done = nf_tables_dump_rules_done,
3744                         .module = THIS_MODULE,
3745                         .data = (void *)nla,
3746                 };
3747
3748                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3749         }
3750
3751         skb2 = nf_tables_getrule_single(portid, info, nla, false);
3752         if (IS_ERR(skb2))
3753                 return PTR_ERR(skb2);
3754
3755         return nfnetlink_unicast(skb2, net, portid);
3756 }
3757
3758 static int nf_tables_getrule_reset(struct sk_buff *skb,
3759                                    const struct nfnl_info *info,
3760                                    const struct nlattr * const nla[])
3761 {
3762         struct nftables_pernet *nft_net = nft_pernet(info->net);
3763         u32 portid = NETLINK_CB(skb).portid;
3764         struct net *net = info->net;
3765         struct sk_buff *skb2;
3766         char *buf;
3767
3768         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3769                 struct netlink_dump_control c = {
3770                         .start= nf_tables_dumpreset_rules_start,
3771                         .dump = nf_tables_dumpreset_rules,
3772                         .done = nf_tables_dump_rules_done,
3773                         .module = THIS_MODULE,
3774                         .data = (void *)nla,
3775                 };
3776
3777                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3778         }
3779
3780         if (!try_module_get(THIS_MODULE))
3781                 return -EINVAL;
3782         rcu_read_unlock();
3783         mutex_lock(&nft_net->commit_mutex);
3784         skb2 = nf_tables_getrule_single(portid, info, nla, true);
3785         mutex_unlock(&nft_net->commit_mutex);
3786         rcu_read_lock();
3787         module_put(THIS_MODULE);
3788
3789         if (IS_ERR(skb2))
3790                 return PTR_ERR(skb2);
3791
3792         buf = kasprintf(GFP_ATOMIC, "%.*s:%u",
3793                         nla_len(nla[NFTA_RULE_TABLE]),
3794                         (char *)nla_data(nla[NFTA_RULE_TABLE]),
3795                         nft_net->base_seq);
3796         audit_log_nfcfg(buf, info->nfmsg->nfgen_family, 1,
3797                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3798         kfree(buf);
3799
3800         return nfnetlink_unicast(skb2, net, portid);
3801 }
3802
3803 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3804 {
3805         struct nft_expr *expr, *next;
3806
3807         /*
3808          * Careful: some expressions might not be initialized in case this
3809          * is called on error from nf_tables_newrule().
3810          */
3811         expr = nft_expr_first(rule);
3812         while (nft_expr_more(rule, expr)) {
3813                 next = nft_expr_next(expr);
3814                 nf_tables_expr_destroy(ctx, expr);
3815                 expr = next;
3816         }
3817         kfree(rule);
3818 }
3819
3820 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3821 {
3822         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3823         nf_tables_rule_destroy(ctx, rule);
3824 }
3825
3826 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3827 {
3828         struct nft_expr *expr, *last;
3829         const struct nft_data *data;
3830         struct nft_rule *rule;
3831         int err;
3832
3833         if (ctx->level == NFT_JUMP_STACK_SIZE)
3834                 return -EMLINK;
3835
3836         list_for_each_entry(rule, &chain->rules, list) {
3837                 if (fatal_signal_pending(current))
3838                         return -EINTR;
3839
3840                 if (!nft_is_active_next(ctx->net, rule))
3841                         continue;
3842
3843                 nft_rule_for_each_expr(expr, last, rule) {
3844                         if (!expr->ops->validate)
3845                                 continue;
3846
3847                         err = expr->ops->validate(ctx, expr, &data);
3848                         if (err < 0)
3849                                 return err;
3850                 }
3851         }
3852
3853         return 0;
3854 }
3855 EXPORT_SYMBOL_GPL(nft_chain_validate);
3856
3857 static int nft_table_validate(struct net *net, const struct nft_table *table)
3858 {
3859         struct nft_chain *chain;
3860         struct nft_ctx ctx = {
3861                 .net    = net,
3862                 .family = table->family,
3863         };
3864         int err;
3865
3866         list_for_each_entry(chain, &table->chains, list) {
3867                 if (!nft_is_base_chain(chain))
3868                         continue;
3869
3870                 ctx.chain = chain;
3871                 err = nft_chain_validate(&ctx, chain);
3872                 if (err < 0)
3873                         return err;
3874
3875                 cond_resched();
3876         }
3877
3878         return 0;
3879 }
3880
3881 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3882                          const struct nft_set_iter *iter,
3883                          struct nft_elem_priv *elem_priv)
3884 {
3885         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
3886         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3887         const struct nft_data *data;
3888         int err;
3889
3890         if (!nft_set_elem_active(ext, iter->genmask))
3891                 return 0;
3892
3893         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3894             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3895                 return 0;
3896
3897         data = nft_set_ext_data(ext);
3898         switch (data->verdict.code) {
3899         case NFT_JUMP:
3900         case NFT_GOTO:
3901                 pctx->level++;
3902                 err = nft_chain_validate(ctx, data->verdict.chain);
3903                 if (err < 0)
3904                         return err;
3905                 pctx->level--;
3906                 break;
3907         default:
3908                 break;
3909         }
3910
3911         return 0;
3912 }
3913
3914 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3915 {
3916         struct nft_set_iter dummy_iter = {
3917                 .genmask        = nft_genmask_next(ctx->net),
3918         };
3919         struct nft_set_elem_catchall *catchall;
3920
3921         struct nft_set_ext *ext;
3922         int ret = 0;
3923
3924         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3925                 ext = nft_set_elem_ext(set, catchall->elem);
3926                 if (!nft_set_elem_active(ext, dummy_iter.genmask))
3927                         continue;
3928
3929                 ret = nft_setelem_validate(ctx, set, &dummy_iter, catchall->elem);
3930                 if (ret < 0)
3931                         return ret;
3932         }
3933
3934         return ret;
3935 }
3936
3937 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3938                                              const struct nft_chain *chain,
3939                                              const struct nlattr *nla);
3940
3941 #define NFT_RULE_MAXEXPRS       128
3942
3943 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3944                              const struct nlattr * const nla[])
3945 {
3946         struct nftables_pernet *nft_net = nft_pernet(info->net);
3947         struct netlink_ext_ack *extack = info->extack;
3948         unsigned int size, i, n, ulen = 0, usize = 0;
3949         u8 genmask = nft_genmask_next(info->net);
3950         struct nft_rule *rule, *old_rule = NULL;
3951         struct nft_expr_info *expr_info = NULL;
3952         u8 family = info->nfmsg->nfgen_family;
3953         struct nft_flow_rule *flow = NULL;
3954         struct net *net = info->net;
3955         struct nft_userdata *udata;
3956         struct nft_table *table;
3957         struct nft_chain *chain;
3958         struct nft_trans *trans;
3959         u64 handle, pos_handle;
3960         struct nft_expr *expr;
3961         struct nft_ctx ctx;
3962         struct nlattr *tmp;
3963         int err, rem;
3964
3965         lockdep_assert_held(&nft_net->commit_mutex);
3966
3967         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3968                                  NETLINK_CB(skb).portid);
3969         if (IS_ERR(table)) {
3970                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3971                 return PTR_ERR(table);
3972         }
3973
3974         if (nla[NFTA_RULE_CHAIN]) {
3975                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3976                                          genmask);
3977                 if (IS_ERR(chain)) {
3978                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3979                         return PTR_ERR(chain);
3980                 }
3981
3982         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3983                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID],
3984                                               genmask);
3985                 if (IS_ERR(chain)) {
3986                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3987                         return PTR_ERR(chain);
3988                 }
3989         } else {
3990                 return -EINVAL;
3991         }
3992
3993         if (nft_chain_is_bound(chain))
3994                 return -EOPNOTSUPP;
3995
3996         if (nla[NFTA_RULE_HANDLE]) {
3997                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3998                 rule = __nft_rule_lookup(chain, handle);
3999                 if (IS_ERR(rule)) {
4000                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4001                         return PTR_ERR(rule);
4002                 }
4003
4004                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4005                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4006                         return -EEXIST;
4007                 }
4008                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4009                         old_rule = rule;
4010                 else
4011                         return -EOPNOTSUPP;
4012         } else {
4013                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
4014                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
4015                         return -EINVAL;
4016                 handle = nf_tables_alloc_handle(table);
4017
4018                 if (nla[NFTA_RULE_POSITION]) {
4019                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
4020                         old_rule = __nft_rule_lookup(chain, pos_handle);
4021                         if (IS_ERR(old_rule)) {
4022                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
4023                                 return PTR_ERR(old_rule);
4024                         }
4025                 } else if (nla[NFTA_RULE_POSITION_ID]) {
4026                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
4027                         if (IS_ERR(old_rule)) {
4028                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
4029                                 return PTR_ERR(old_rule);
4030                         }
4031                 }
4032         }
4033
4034         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4035
4036         n = 0;
4037         size = 0;
4038         if (nla[NFTA_RULE_EXPRESSIONS]) {
4039                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
4040                                            sizeof(struct nft_expr_info),
4041                                            GFP_KERNEL);
4042                 if (!expr_info)
4043                         return -ENOMEM;
4044
4045                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
4046                         err = -EINVAL;
4047                         if (nla_type(tmp) != NFTA_LIST_ELEM)
4048                                 goto err_release_expr;
4049                         if (n == NFT_RULE_MAXEXPRS)
4050                                 goto err_release_expr;
4051                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
4052                         if (err < 0) {
4053                                 NL_SET_BAD_ATTR(extack, tmp);
4054                                 goto err_release_expr;
4055                         }
4056                         size += expr_info[n].ops->size;
4057                         n++;
4058                 }
4059         }
4060         /* Check for overflow of dlen field */
4061         err = -EFBIG;
4062         if (size >= 1 << 12)
4063                 goto err_release_expr;
4064
4065         if (nla[NFTA_RULE_USERDATA]) {
4066                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
4067                 if (ulen > 0)
4068                         usize = sizeof(struct nft_userdata) + ulen;
4069         }
4070
4071         err = -ENOMEM;
4072         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
4073         if (rule == NULL)
4074                 goto err_release_expr;
4075
4076         nft_activate_next(net, rule);
4077
4078         rule->handle = handle;
4079         rule->dlen   = size;
4080         rule->udata  = ulen ? 1 : 0;
4081
4082         if (ulen) {
4083                 udata = nft_userdata(rule);
4084                 udata->len = ulen - 1;
4085                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
4086         }
4087
4088         expr = nft_expr_first(rule);
4089         for (i = 0; i < n; i++) {
4090                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
4091                 if (err < 0) {
4092                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
4093                         goto err_release_rule;
4094                 }
4095
4096                 if (expr_info[i].ops->validate)
4097                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
4098
4099                 expr_info[i].ops = NULL;
4100                 expr = nft_expr_next(expr);
4101         }
4102
4103         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
4104                 flow = nft_flow_rule_create(net, rule);
4105                 if (IS_ERR(flow)) {
4106                         err = PTR_ERR(flow);
4107                         goto err_release_rule;
4108                 }
4109         }
4110
4111         if (!nft_use_inc(&chain->use)) {
4112                 err = -EMFILE;
4113                 goto err_release_rule;
4114         }
4115
4116         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
4117                 if (nft_chain_binding(chain)) {
4118                         err = -EOPNOTSUPP;
4119                         goto err_destroy_flow_rule;
4120                 }
4121
4122                 err = nft_delrule(&ctx, old_rule);
4123                 if (err < 0)
4124                         goto err_destroy_flow_rule;
4125
4126                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4127                 if (trans == NULL) {
4128                         err = -ENOMEM;
4129                         goto err_destroy_flow_rule;
4130                 }
4131                 list_add_tail_rcu(&rule->list, &old_rule->list);
4132         } else {
4133                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4134                 if (!trans) {
4135                         err = -ENOMEM;
4136                         goto err_destroy_flow_rule;
4137                 }
4138
4139                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
4140                         if (old_rule)
4141                                 list_add_rcu(&rule->list, &old_rule->list);
4142                         else
4143                                 list_add_tail_rcu(&rule->list, &chain->rules);
4144                  } else {
4145                         if (old_rule)
4146                                 list_add_tail_rcu(&rule->list, &old_rule->list);
4147                         else
4148                                 list_add_rcu(&rule->list, &chain->rules);
4149                 }
4150         }
4151         kvfree(expr_info);
4152
4153         if (flow)
4154                 nft_trans_flow_rule(trans) = flow;
4155
4156         if (table->validate_state == NFT_VALIDATE_DO)
4157                 return nft_table_validate(net, table);
4158
4159         return 0;
4160
4161 err_destroy_flow_rule:
4162         nft_use_dec_restore(&chain->use);
4163         if (flow)
4164                 nft_flow_rule_destroy(flow);
4165 err_release_rule:
4166         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
4167         nf_tables_rule_destroy(&ctx, rule);
4168 err_release_expr:
4169         for (i = 0; i < n; i++) {
4170                 if (expr_info[i].ops) {
4171                         module_put(expr_info[i].ops->type->owner);
4172                         if (expr_info[i].ops->type->release_ops)
4173                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
4174                 }
4175         }
4176         kvfree(expr_info);
4177
4178         return err;
4179 }
4180
4181 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
4182                                              const struct nft_chain *chain,
4183                                              const struct nlattr *nla)
4184 {
4185         struct nftables_pernet *nft_net = nft_pernet(net);
4186         u32 id = ntohl(nla_get_be32(nla));
4187         struct nft_trans *trans;
4188
4189         list_for_each_entry(trans, &nft_net->commit_list, list) {
4190                 if (trans->msg_type == NFT_MSG_NEWRULE &&
4191                     trans->ctx.chain == chain &&
4192                     id == nft_trans_rule_id(trans))
4193                         return nft_trans_rule(trans);
4194         }
4195         return ERR_PTR(-ENOENT);
4196 }
4197
4198 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
4199                              const struct nlattr * const nla[])
4200 {
4201         struct netlink_ext_ack *extack = info->extack;
4202         u8 genmask = nft_genmask_next(info->net);
4203         u8 family = info->nfmsg->nfgen_family;
4204         struct nft_chain *chain = NULL;
4205         struct net *net = info->net;
4206         struct nft_table *table;
4207         struct nft_rule *rule;
4208         struct nft_ctx ctx;
4209         int err = 0;
4210
4211         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4212                                  NETLINK_CB(skb).portid);
4213         if (IS_ERR(table)) {
4214                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4215                 return PTR_ERR(table);
4216         }
4217
4218         if (nla[NFTA_RULE_CHAIN]) {
4219                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4220                                          genmask);
4221                 if (IS_ERR(chain)) {
4222                         if (PTR_ERR(chain) == -ENOENT &&
4223                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4224                                 return 0;
4225
4226                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4227                         return PTR_ERR(chain);
4228                 }
4229                 if (nft_chain_binding(chain))
4230                         return -EOPNOTSUPP;
4231         }
4232
4233         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4234
4235         if (chain) {
4236                 if (nla[NFTA_RULE_HANDLE]) {
4237                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
4238                         if (IS_ERR(rule)) {
4239                                 if (PTR_ERR(rule) == -ENOENT &&
4240                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4241                                         return 0;
4242
4243                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4244                                 return PTR_ERR(rule);
4245                         }
4246
4247                         err = nft_delrule(&ctx, rule);
4248                 } else if (nla[NFTA_RULE_ID]) {
4249                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
4250                         if (IS_ERR(rule)) {
4251                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
4252                                 return PTR_ERR(rule);
4253                         }
4254
4255                         err = nft_delrule(&ctx, rule);
4256                 } else {
4257                         err = nft_delrule_by_chain(&ctx);
4258                 }
4259         } else {
4260                 list_for_each_entry(chain, &table->chains, list) {
4261                         if (!nft_is_active_next(net, chain))
4262                                 continue;
4263                         if (nft_chain_binding(chain))
4264                                 continue;
4265
4266                         ctx.chain = chain;
4267                         err = nft_delrule_by_chain(&ctx);
4268                         if (err < 0)
4269                                 break;
4270                 }
4271         }
4272
4273         return err;
4274 }
4275
4276 /*
4277  * Sets
4278  */
4279 static const struct nft_set_type *nft_set_types[] = {
4280         &nft_set_hash_fast_type,
4281         &nft_set_hash_type,
4282         &nft_set_rhash_type,
4283         &nft_set_bitmap_type,
4284         &nft_set_rbtree_type,
4285 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4286         &nft_set_pipapo_avx2_type,
4287 #endif
4288         &nft_set_pipapo_type,
4289 };
4290
4291 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4292                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4293                                  NFT_SET_EVAL)
4294
4295 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4296 {
4297         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4298 }
4299
4300 /*
4301  * Select a set implementation based on the data characteristics and the
4302  * given policy. The total memory use might not be known if no size is
4303  * given, in that case the amount of memory per element is used.
4304  */
4305 static const struct nft_set_ops *
4306 nft_select_set_ops(const struct nft_ctx *ctx, u32 flags,
4307                    const struct nft_set_desc *desc)
4308 {
4309         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4310         const struct nft_set_ops *ops, *bops;
4311         struct nft_set_estimate est, best;
4312         const struct nft_set_type *type;
4313         int i;
4314
4315         lockdep_assert_held(&nft_net->commit_mutex);
4316         lockdep_nfnl_nft_mutex_not_held();
4317
4318         bops        = NULL;
4319         best.size   = ~0;
4320         best.lookup = ~0;
4321         best.space  = ~0;
4322
4323         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4324                 type = nft_set_types[i];
4325                 ops = &type->ops;
4326
4327                 if (!nft_set_ops_candidate(type, flags))
4328                         continue;
4329                 if (!ops->estimate(desc, flags, &est))
4330                         continue;
4331
4332                 switch (desc->policy) {
4333                 case NFT_SET_POL_PERFORMANCE:
4334                         if (est.lookup < best.lookup)
4335                                 break;
4336                         if (est.lookup == best.lookup &&
4337                             est.space < best.space)
4338                                 break;
4339                         continue;
4340                 case NFT_SET_POL_MEMORY:
4341                         if (!desc->size) {
4342                                 if (est.space < best.space)
4343                                         break;
4344                                 if (est.space == best.space &&
4345                                     est.lookup < best.lookup)
4346                                         break;
4347                         } else if (est.size < best.size || !bops) {
4348                                 break;
4349                         }
4350                         continue;
4351                 default:
4352                         break;
4353                 }
4354
4355                 bops = ops;
4356                 best = est;
4357         }
4358
4359         if (bops != NULL)
4360                 return bops;
4361
4362         return ERR_PTR(-EOPNOTSUPP);
4363 }
4364
4365 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4366         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4367                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4368         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4369                                             .len = NFT_SET_MAXNAMELEN - 1 },
4370         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4371         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4372         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4373         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4374         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4375         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4376         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4377         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4378         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4379         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4380         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4381                                             .len  = NFT_USERDATA_MAXLEN },
4382         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4383         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4384         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4385         [NFTA_SET_EXPRESSIONS]          = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
4386 };
4387
4388 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4389         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4390 };
4391
4392 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4393         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4394         [NFTA_SET_DESC_CONCAT]          = NLA_POLICY_NESTED_ARRAY(nft_concat_policy),
4395 };
4396
4397 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4398                                       const struct nlattr *nla, u8 genmask)
4399 {
4400         struct nft_set *set;
4401
4402         if (nla == NULL)
4403                 return ERR_PTR(-EINVAL);
4404
4405         list_for_each_entry_rcu(set, &table->sets, list) {
4406                 if (!nla_strcmp(nla, set->name) &&
4407                     nft_active_genmask(set, genmask))
4408                         return set;
4409         }
4410         return ERR_PTR(-ENOENT);
4411 }
4412
4413 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4414                                                const struct nlattr *nla,
4415                                                u8 genmask)
4416 {
4417         struct nft_set *set;
4418
4419         list_for_each_entry(set, &table->sets, list) {
4420                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4421                     nft_active_genmask(set, genmask))
4422                         return set;
4423         }
4424         return ERR_PTR(-ENOENT);
4425 }
4426
4427 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4428                                            const struct nft_table *table,
4429                                            const struct nlattr *nla, u8 genmask)
4430 {
4431         struct nftables_pernet *nft_net = nft_pernet(net);
4432         u32 id = ntohl(nla_get_be32(nla));
4433         struct nft_trans *trans;
4434
4435         list_for_each_entry(trans, &nft_net->commit_list, list) {
4436                 if (trans->msg_type == NFT_MSG_NEWSET) {
4437                         struct nft_set *set = nft_trans_set(trans);
4438
4439                         if (id == nft_trans_set_id(trans) &&
4440                             set->table == table &&
4441                             nft_active_genmask(set, genmask))
4442                                 return set;
4443                 }
4444         }
4445         return ERR_PTR(-ENOENT);
4446 }
4447
4448 struct nft_set *nft_set_lookup_global(const struct net *net,
4449                                       const struct nft_table *table,
4450                                       const struct nlattr *nla_set_name,
4451                                       const struct nlattr *nla_set_id,
4452                                       u8 genmask)
4453 {
4454         struct nft_set *set;
4455
4456         set = nft_set_lookup(table, nla_set_name, genmask);
4457         if (IS_ERR(set)) {
4458                 if (!nla_set_id)
4459                         return set;
4460
4461                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4462         }
4463         return set;
4464 }
4465 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4466
4467 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4468                                     const char *name)
4469 {
4470         const struct nft_set *i;
4471         const char *p;
4472         unsigned long *inuse;
4473         unsigned int n = 0, min = 0;
4474
4475         p = strchr(name, '%');
4476         if (p != NULL) {
4477                 if (p[1] != 'd' || strchr(p + 2, '%'))
4478                         return -EINVAL;
4479
4480                 if (strnlen(name, NFT_SET_MAX_ANONLEN) >= NFT_SET_MAX_ANONLEN)
4481                         return -EINVAL;
4482
4483                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4484                 if (inuse == NULL)
4485                         return -ENOMEM;
4486 cont:
4487                 list_for_each_entry(i, &ctx->table->sets, list) {
4488                         int tmp;
4489
4490                         if (!nft_is_active_next(ctx->net, i))
4491                                 continue;
4492                         if (!sscanf(i->name, name, &tmp))
4493                                 continue;
4494                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4495                                 continue;
4496
4497                         set_bit(tmp - min, inuse);
4498                 }
4499
4500                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4501                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4502                         min += BITS_PER_BYTE * PAGE_SIZE;
4503                         memset(inuse, 0, PAGE_SIZE);
4504                         goto cont;
4505                 }
4506                 free_page((unsigned long)inuse);
4507         }
4508
4509         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4510         if (!set->name)
4511                 return -ENOMEM;
4512
4513         list_for_each_entry(i, &ctx->table->sets, list) {
4514                 if (!nft_is_active_next(ctx->net, i))
4515                         continue;
4516                 if (!strcmp(set->name, i->name)) {
4517                         kfree(set->name);
4518                         set->name = NULL;
4519                         return -ENFILE;
4520                 }
4521         }
4522         return 0;
4523 }
4524
4525 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4526 {
4527         u64 ms = be64_to_cpu(nla_get_be64(nla));
4528         u64 max = (u64)(~((u64)0));
4529
4530         max = div_u64(max, NSEC_PER_MSEC);
4531         if (ms >= max)
4532                 return -ERANGE;
4533
4534         ms *= NSEC_PER_MSEC;
4535         *result = nsecs_to_jiffies64(ms);
4536         return 0;
4537 }
4538
4539 __be64 nf_jiffies64_to_msecs(u64 input)
4540 {
4541         return cpu_to_be64(jiffies64_to_msecs(input));
4542 }
4543
4544 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4545                                      const struct nft_set *set)
4546 {
4547         struct nlattr *concat, *field;
4548         int i;
4549
4550         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4551         if (!concat)
4552                 return -ENOMEM;
4553
4554         for (i = 0; i < set->field_count; i++) {
4555                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4556                 if (!field)
4557                         return -ENOMEM;
4558
4559                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4560                                  htonl(set->field_len[i])))
4561                         return -ENOMEM;
4562
4563                 nla_nest_end(skb, field);
4564         }
4565
4566         nla_nest_end(skb, concat);
4567
4568         return 0;
4569 }
4570
4571 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4572                               const struct nft_set *set, u16 event, u16 flags)
4573 {
4574         u64 timeout = READ_ONCE(set->timeout);
4575         u32 gc_int = READ_ONCE(set->gc_int);
4576         u32 portid = ctx->portid;
4577         struct nlmsghdr *nlh;
4578         struct nlattr *nest;
4579         u32 seq = ctx->seq;
4580         int i;
4581
4582         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4583         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4584                            NFNETLINK_V0, nft_base_seq(ctx->net));
4585         if (!nlh)
4586                 goto nla_put_failure;
4587
4588         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4589                 goto nla_put_failure;
4590         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4591                 goto nla_put_failure;
4592         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4593                          NFTA_SET_PAD))
4594                 goto nla_put_failure;
4595
4596         if (event == NFT_MSG_DELSET) {
4597                 nlmsg_end(skb, nlh);
4598                 return 0;
4599         }
4600
4601         if (set->flags != 0)
4602                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4603                         goto nla_put_failure;
4604
4605         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4606                 goto nla_put_failure;
4607         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4608                 goto nla_put_failure;
4609         if (set->flags & NFT_SET_MAP) {
4610                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4611                         goto nla_put_failure;
4612                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4613                         goto nla_put_failure;
4614         }
4615         if (set->flags & NFT_SET_OBJECT &&
4616             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4617                 goto nla_put_failure;
4618
4619         if (timeout &&
4620             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4621                          nf_jiffies64_to_msecs(timeout),
4622                          NFTA_SET_PAD))
4623                 goto nla_put_failure;
4624         if (gc_int &&
4625             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4626                 goto nla_put_failure;
4627
4628         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4629                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4630                         goto nla_put_failure;
4631         }
4632
4633         if (set->udata &&
4634             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4635                 goto nla_put_failure;
4636
4637         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4638         if (!nest)
4639                 goto nla_put_failure;
4640         if (set->size &&
4641             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4642                 goto nla_put_failure;
4643
4644         if (set->field_count > 1 &&
4645             nf_tables_fill_set_concat(skb, set))
4646                 goto nla_put_failure;
4647
4648         nla_nest_end(skb, nest);
4649
4650         if (set->num_exprs == 1) {
4651                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4652                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4653                         goto nla_put_failure;
4654
4655                 nla_nest_end(skb, nest);
4656         } else if (set->num_exprs > 1) {
4657                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4658                 if (nest == NULL)
4659                         goto nla_put_failure;
4660
4661                 for (i = 0; i < set->num_exprs; i++) {
4662                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4663                                           set->exprs[i], false) < 0)
4664                                 goto nla_put_failure;
4665                 }
4666                 nla_nest_end(skb, nest);
4667         }
4668
4669         nlmsg_end(skb, nlh);
4670         return 0;
4671
4672 nla_put_failure:
4673         nlmsg_trim(skb, nlh);
4674         return -1;
4675 }
4676
4677 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4678                                  const struct nft_set *set, int event,
4679                                  gfp_t gfp_flags)
4680 {
4681         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4682         u32 portid = ctx->portid;
4683         struct sk_buff *skb;
4684         u16 flags = 0;
4685         int err;
4686
4687         if (!ctx->report &&
4688             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4689                 return;
4690
4691         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4692         if (skb == NULL)
4693                 goto err;
4694
4695         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4696                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4697
4698         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4699         if (err < 0) {
4700                 kfree_skb(skb);
4701                 goto err;
4702         }
4703
4704         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4705         return;
4706 err:
4707         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4708 }
4709
4710 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4711 {
4712         const struct nft_set *set;
4713         unsigned int idx, s_idx = cb->args[0];
4714         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4715         struct net *net = sock_net(skb->sk);
4716         struct nft_ctx *ctx = cb->data, ctx_set;
4717         struct nftables_pernet *nft_net;
4718
4719         if (cb->args[1])
4720                 return skb->len;
4721
4722         rcu_read_lock();
4723         nft_net = nft_pernet(net);
4724         cb->seq = READ_ONCE(nft_net->base_seq);
4725
4726         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4727                 if (ctx->family != NFPROTO_UNSPEC &&
4728                     ctx->family != table->family)
4729                         continue;
4730
4731                 if (ctx->table && ctx->table != table)
4732                         continue;
4733
4734                 if (cur_table) {
4735                         if (cur_table != table)
4736                                 continue;
4737
4738                         cur_table = NULL;
4739                 }
4740                 idx = 0;
4741                 list_for_each_entry_rcu(set, &table->sets, list) {
4742                         if (idx < s_idx)
4743                                 goto cont;
4744                         if (!nft_is_active(net, set))
4745                                 goto cont;
4746
4747                         ctx_set = *ctx;
4748                         ctx_set.table = table;
4749                         ctx_set.family = table->family;
4750
4751                         if (nf_tables_fill_set(skb, &ctx_set, set,
4752                                                NFT_MSG_NEWSET,
4753                                                NLM_F_MULTI) < 0) {
4754                                 cb->args[0] = idx;
4755                                 cb->args[2] = (unsigned long) table;
4756                                 goto done;
4757                         }
4758                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4759 cont:
4760                         idx++;
4761                 }
4762                 if (s_idx)
4763                         s_idx = 0;
4764         }
4765         cb->args[1] = 1;
4766 done:
4767         rcu_read_unlock();
4768         return skb->len;
4769 }
4770
4771 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4772 {
4773         struct nft_ctx *ctx_dump = NULL;
4774
4775         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4776         if (ctx_dump == NULL)
4777                 return -ENOMEM;
4778
4779         cb->data = ctx_dump;
4780         return 0;
4781 }
4782
4783 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4784 {
4785         kfree(cb->data);
4786         return 0;
4787 }
4788
4789 /* called with rcu_read_lock held */
4790 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4791                             const struct nlattr * const nla[])
4792 {
4793         struct netlink_ext_ack *extack = info->extack;
4794         u8 genmask = nft_genmask_cur(info->net);
4795         u8 family = info->nfmsg->nfgen_family;
4796         struct nft_table *table = NULL;
4797         struct net *net = info->net;
4798         const struct nft_set *set;
4799         struct sk_buff *skb2;
4800         struct nft_ctx ctx;
4801         int err;
4802
4803         if (nla[NFTA_SET_TABLE]) {
4804                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4805                                          genmask, 0);
4806                 if (IS_ERR(table)) {
4807                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4808                         return PTR_ERR(table);
4809                 }
4810         }
4811
4812         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4813
4814         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4815                 struct netlink_dump_control c = {
4816                         .start = nf_tables_dump_sets_start,
4817                         .dump = nf_tables_dump_sets,
4818                         .done = nf_tables_dump_sets_done,
4819                         .data = &ctx,
4820                         .module = THIS_MODULE,
4821                 };
4822
4823                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4824         }
4825
4826         /* Only accept unspec with dump */
4827         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4828                 return -EAFNOSUPPORT;
4829         if (!nla[NFTA_SET_TABLE])
4830                 return -EINVAL;
4831
4832         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4833         if (IS_ERR(set)) {
4834                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4835                 return PTR_ERR(set);
4836         }
4837
4838         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4839         if (skb2 == NULL)
4840                 return -ENOMEM;
4841
4842         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4843         if (err < 0)
4844                 goto err_fill_set_info;
4845
4846         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4847
4848 err_fill_set_info:
4849         kfree_skb(skb2);
4850         return err;
4851 }
4852
4853 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4854                                      struct nft_set_desc *desc)
4855 {
4856         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4857         u32 len;
4858         int err;
4859
4860         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4861                 return -E2BIG;
4862
4863         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4864                                           nft_concat_policy, NULL);
4865         if (err < 0)
4866                 return err;
4867
4868         if (!tb[NFTA_SET_FIELD_LEN])
4869                 return -EINVAL;
4870
4871         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4872         if (!len || len > U8_MAX)
4873                 return -EINVAL;
4874
4875         desc->field_len[desc->field_count++] = len;
4876
4877         return 0;
4878 }
4879
4880 static int nft_set_desc_concat(struct nft_set_desc *desc,
4881                                const struct nlattr *nla)
4882 {
4883         u32 num_regs = 0, key_num_regs = 0;
4884         struct nlattr *attr;
4885         int rem, err, i;
4886
4887         nla_for_each_nested(attr, nla, rem) {
4888                 if (nla_type(attr) != NFTA_LIST_ELEM)
4889                         return -EINVAL;
4890
4891                 err = nft_set_desc_concat_parse(attr, desc);
4892                 if (err < 0)
4893                         return err;
4894         }
4895
4896         for (i = 0; i < desc->field_count; i++)
4897                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4898
4899         key_num_regs = DIV_ROUND_UP(desc->klen, sizeof(u32));
4900         if (key_num_regs != num_regs)
4901                 return -EINVAL;
4902
4903         if (num_regs > NFT_REG32_COUNT)
4904                 return -E2BIG;
4905
4906         return 0;
4907 }
4908
4909 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4910                                     const struct nlattr *nla)
4911 {
4912         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4913         int err;
4914
4915         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4916                                           nft_set_desc_policy, NULL);
4917         if (err < 0)
4918                 return err;
4919
4920         if (da[NFTA_SET_DESC_SIZE] != NULL)
4921                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4922         if (da[NFTA_SET_DESC_CONCAT])
4923                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4924
4925         return err;
4926 }
4927
4928 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4929                               const struct nlattr * const *nla,
4930                               struct nft_expr **exprs, int *num_exprs,
4931                               u32 flags)
4932 {
4933         struct nft_expr *expr;
4934         int err, i;
4935
4936         if (nla[NFTA_SET_EXPR]) {
4937                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4938                 if (IS_ERR(expr)) {
4939                         err = PTR_ERR(expr);
4940                         goto err_set_expr_alloc;
4941                 }
4942                 exprs[0] = expr;
4943                 (*num_exprs)++;
4944         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4945                 struct nlattr *tmp;
4946                 int left;
4947
4948                 if (!(flags & NFT_SET_EXPR)) {
4949                         err = -EINVAL;
4950                         goto err_set_expr_alloc;
4951                 }
4952                 i = 0;
4953                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4954                         if (i == NFT_SET_EXPR_MAX) {
4955                                 err = -E2BIG;
4956                                 goto err_set_expr_alloc;
4957                         }
4958                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4959                                 err = -EINVAL;
4960                                 goto err_set_expr_alloc;
4961                         }
4962                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4963                         if (IS_ERR(expr)) {
4964                                 err = PTR_ERR(expr);
4965                                 goto err_set_expr_alloc;
4966                         }
4967                         exprs[i++] = expr;
4968                         (*num_exprs)++;
4969                 }
4970         }
4971
4972         return 0;
4973
4974 err_set_expr_alloc:
4975         for (i = 0; i < *num_exprs; i++)
4976                 nft_expr_destroy(ctx, exprs[i]);
4977
4978         return err;
4979 }
4980
4981 static bool nft_set_is_same(const struct nft_set *set,
4982                             const struct nft_set_desc *desc,
4983                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4984 {
4985         int i;
4986
4987         if (set->ktype != desc->ktype ||
4988             set->dtype != desc->dtype ||
4989             set->flags != flags ||
4990             set->klen != desc->klen ||
4991             set->dlen != desc->dlen ||
4992             set->field_count != desc->field_count ||
4993             set->num_exprs != num_exprs)
4994                 return false;
4995
4996         for (i = 0; i < desc->field_count; i++) {
4997                 if (set->field_len[i] != desc->field_len[i])
4998                         return false;
4999         }
5000
5001         for (i = 0; i < num_exprs; i++) {
5002                 if (set->exprs[i]->ops != exprs[i]->ops)
5003                         return false;
5004         }
5005
5006         return true;
5007 }
5008
5009 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
5010                             const struct nlattr * const nla[])
5011 {
5012         struct netlink_ext_ack *extack = info->extack;
5013         u8 genmask = nft_genmask_next(info->net);
5014         u8 family = info->nfmsg->nfgen_family;
5015         const struct nft_set_ops *ops;
5016         struct net *net = info->net;
5017         struct nft_set_desc desc;
5018         struct nft_table *table;
5019         unsigned char *udata;
5020         struct nft_set *set;
5021         struct nft_ctx ctx;
5022         size_t alloc_size;
5023         int num_exprs = 0;
5024         char *name;
5025         int err, i;
5026         u16 udlen;
5027         u32 flags;
5028         u64 size;
5029
5030         if (nla[NFTA_SET_TABLE] == NULL ||
5031             nla[NFTA_SET_NAME] == NULL ||
5032             nla[NFTA_SET_KEY_LEN] == NULL ||
5033             nla[NFTA_SET_ID] == NULL)
5034                 return -EINVAL;
5035
5036         memset(&desc, 0, sizeof(desc));
5037
5038         desc.ktype = NFT_DATA_VALUE;
5039         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
5040                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
5041                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
5042                         return -EINVAL;
5043         }
5044
5045         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
5046         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
5047                 return -EINVAL;
5048
5049         flags = 0;
5050         if (nla[NFTA_SET_FLAGS] != NULL) {
5051                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
5052                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
5053                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
5054                               NFT_SET_MAP | NFT_SET_EVAL |
5055                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
5056                         return -EOPNOTSUPP;
5057                 /* Only one of these operations is supported */
5058                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
5059                              (NFT_SET_MAP | NFT_SET_OBJECT))
5060                         return -EOPNOTSUPP;
5061                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
5062                              (NFT_SET_EVAL | NFT_SET_OBJECT))
5063                         return -EOPNOTSUPP;
5064                 if ((flags & (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT | NFT_SET_EVAL)) ==
5065                              (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT))
5066                         return -EOPNOTSUPP;
5067                 if ((flags & (NFT_SET_CONSTANT | NFT_SET_TIMEOUT)) ==
5068                              (NFT_SET_CONSTANT | NFT_SET_TIMEOUT))
5069                         return -EOPNOTSUPP;
5070         }
5071
5072         desc.dtype = 0;
5073         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
5074                 if (!(flags & NFT_SET_MAP))
5075                         return -EINVAL;
5076
5077                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
5078                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
5079                     desc.dtype != NFT_DATA_VERDICT)
5080                         return -EINVAL;
5081
5082                 if (desc.dtype != NFT_DATA_VERDICT) {
5083                         if (nla[NFTA_SET_DATA_LEN] == NULL)
5084                                 return -EINVAL;
5085                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
5086                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
5087                                 return -EINVAL;
5088                 } else
5089                         desc.dlen = sizeof(struct nft_verdict);
5090         } else if (flags & NFT_SET_MAP)
5091                 return -EINVAL;
5092
5093         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
5094                 if (!(flags & NFT_SET_OBJECT))
5095                         return -EINVAL;
5096
5097                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
5098                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
5099                     desc.objtype > NFT_OBJECT_MAX)
5100                         return -EOPNOTSUPP;
5101         } else if (flags & NFT_SET_OBJECT)
5102                 return -EINVAL;
5103         else
5104                 desc.objtype = NFT_OBJECT_UNSPEC;
5105
5106         desc.timeout = 0;
5107         if (nla[NFTA_SET_TIMEOUT] != NULL) {
5108                 if (!(flags & NFT_SET_TIMEOUT))
5109                         return -EINVAL;
5110
5111                 if (flags & NFT_SET_ANONYMOUS)
5112                         return -EOPNOTSUPP;
5113
5114                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
5115                 if (err)
5116                         return err;
5117         }
5118         desc.gc_int = 0;
5119         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
5120                 if (!(flags & NFT_SET_TIMEOUT))
5121                         return -EINVAL;
5122
5123                 if (flags & NFT_SET_ANONYMOUS)
5124                         return -EOPNOTSUPP;
5125
5126                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
5127         }
5128
5129         desc.policy = NFT_SET_POL_PERFORMANCE;
5130         if (nla[NFTA_SET_POLICY] != NULL) {
5131                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
5132                 switch (desc.policy) {
5133                 case NFT_SET_POL_PERFORMANCE:
5134                 case NFT_SET_POL_MEMORY:
5135                         break;
5136                 default:
5137                         return -EOPNOTSUPP;
5138                 }
5139         }
5140
5141         if (nla[NFTA_SET_DESC] != NULL) {
5142                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
5143                 if (err < 0)
5144                         return err;
5145
5146                 if (desc.field_count > 1) {
5147                         if (!(flags & NFT_SET_CONCAT))
5148                                 return -EINVAL;
5149                 } else if (flags & NFT_SET_CONCAT) {
5150                         return -EINVAL;
5151                 }
5152         } else if (flags & NFT_SET_CONCAT) {
5153                 return -EINVAL;
5154         }
5155
5156         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
5157                 desc.expr = true;
5158
5159         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
5160                                  NETLINK_CB(skb).portid);
5161         if (IS_ERR(table)) {
5162                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5163                 return PTR_ERR(table);
5164         }
5165
5166         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5167
5168         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
5169         if (IS_ERR(set)) {
5170                 if (PTR_ERR(set) != -ENOENT) {
5171                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5172                         return PTR_ERR(set);
5173                 }
5174         } else {
5175                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
5176
5177                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
5178                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5179                         return -EEXIST;
5180                 }
5181                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
5182                         return -EOPNOTSUPP;
5183
5184                 if (nft_set_is_anonymous(set))
5185                         return -EOPNOTSUPP;
5186
5187                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
5188                 if (err < 0)
5189                         return err;
5190
5191                 err = 0;
5192                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
5193                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5194                         err = -EEXIST;
5195                 }
5196
5197                 for (i = 0; i < num_exprs; i++)
5198                         nft_expr_destroy(&ctx, exprs[i]);
5199
5200                 if (err < 0)
5201                         return err;
5202
5203                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
5204         }
5205
5206         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
5207                 return -ENOENT;
5208
5209         ops = nft_select_set_ops(&ctx, flags, &desc);
5210         if (IS_ERR(ops))
5211                 return PTR_ERR(ops);
5212
5213         udlen = 0;
5214         if (nla[NFTA_SET_USERDATA])
5215                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
5216
5217         size = 0;
5218         if (ops->privsize != NULL)
5219                 size = ops->privsize(nla, &desc);
5220         alloc_size = sizeof(*set) + size + udlen;
5221         if (alloc_size < size || alloc_size > INT_MAX)
5222                 return -ENOMEM;
5223
5224         if (!nft_use_inc(&table->use))
5225                 return -EMFILE;
5226
5227         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
5228         if (!set) {
5229                 err = -ENOMEM;
5230                 goto err_alloc;
5231         }
5232
5233         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
5234         if (!name) {
5235                 err = -ENOMEM;
5236                 goto err_set_name;
5237         }
5238
5239         err = nf_tables_set_alloc_name(&ctx, set, name);
5240         kfree(name);
5241         if (err < 0)
5242                 goto err_set_name;
5243
5244         udata = NULL;
5245         if (udlen) {
5246                 udata = set->data + size;
5247                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
5248         }
5249
5250         INIT_LIST_HEAD(&set->bindings);
5251         INIT_LIST_HEAD(&set->catchall_list);
5252         refcount_set(&set->refs, 1);
5253         set->table = table;
5254         write_pnet(&set->net, net);
5255         set->ops = ops;
5256         set->ktype = desc.ktype;
5257         set->klen = desc.klen;
5258         set->dtype = desc.dtype;
5259         set->objtype = desc.objtype;
5260         set->dlen = desc.dlen;
5261         set->flags = flags;
5262         set->size = desc.size;
5263         set->policy = desc.policy;
5264         set->udlen = udlen;
5265         set->udata = udata;
5266         set->timeout = desc.timeout;
5267         set->gc_int = desc.gc_int;
5268
5269         set->field_count = desc.field_count;
5270         for (i = 0; i < desc.field_count; i++)
5271                 set->field_len[i] = desc.field_len[i];
5272
5273         err = ops->init(set, &desc, nla);
5274         if (err < 0)
5275                 goto err_set_init;
5276
5277         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
5278         if (err < 0)
5279                 goto err_set_destroy;
5280
5281         set->num_exprs = num_exprs;
5282         set->handle = nf_tables_alloc_handle(table);
5283         INIT_LIST_HEAD(&set->pending_update);
5284
5285         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
5286         if (err < 0)
5287                 goto err_set_expr_alloc;
5288
5289         list_add_tail_rcu(&set->list, &table->sets);
5290
5291         return 0;
5292
5293 err_set_expr_alloc:
5294         for (i = 0; i < set->num_exprs; i++)
5295                 nft_expr_destroy(&ctx, set->exprs[i]);
5296 err_set_destroy:
5297         ops->destroy(&ctx, set);
5298 err_set_init:
5299         kfree(set->name);
5300 err_set_name:
5301         kvfree(set);
5302 err_alloc:
5303         nft_use_dec_restore(&table->use);
5304
5305         return err;
5306 }
5307
5308 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5309                                      struct nft_set *set)
5310 {
5311         struct nft_set_elem_catchall *next, *catchall;
5312
5313         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5314                 list_del_rcu(&catchall->list);
5315                 nf_tables_set_elem_destroy(ctx, set, catchall->elem);
5316                 kfree_rcu(catchall, rcu);
5317         }
5318 }
5319
5320 static void nft_set_put(struct nft_set *set)
5321 {
5322         if (refcount_dec_and_test(&set->refs)) {
5323                 kfree(set->name);
5324                 kvfree(set);
5325         }
5326 }
5327
5328 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5329 {
5330         int i;
5331
5332         if (WARN_ON(set->use > 0))
5333                 return;
5334
5335         for (i = 0; i < set->num_exprs; i++)
5336                 nft_expr_destroy(ctx, set->exprs[i]);
5337
5338         set->ops->destroy(ctx, set);
5339         nft_set_catchall_destroy(ctx, set);
5340         nft_set_put(set);
5341 }
5342
5343 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5344                             const struct nlattr * const nla[])
5345 {
5346         struct netlink_ext_ack *extack = info->extack;
5347         u8 genmask = nft_genmask_next(info->net);
5348         u8 family = info->nfmsg->nfgen_family;
5349         struct net *net = info->net;
5350         const struct nlattr *attr;
5351         struct nft_table *table;
5352         struct nft_set *set;
5353         struct nft_ctx ctx;
5354
5355         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5356                 return -EAFNOSUPPORT;
5357
5358         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5359                                  genmask, NETLINK_CB(skb).portid);
5360         if (IS_ERR(table)) {
5361                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5362                 return PTR_ERR(table);
5363         }
5364
5365         if (nla[NFTA_SET_HANDLE]) {
5366                 attr = nla[NFTA_SET_HANDLE];
5367                 set = nft_set_lookup_byhandle(table, attr, genmask);
5368         } else {
5369                 attr = nla[NFTA_SET_NAME];
5370                 set = nft_set_lookup(table, attr, genmask);
5371         }
5372
5373         if (IS_ERR(set)) {
5374                 if (PTR_ERR(set) == -ENOENT &&
5375                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5376                         return 0;
5377
5378                 NL_SET_BAD_ATTR(extack, attr);
5379                 return PTR_ERR(set);
5380         }
5381         if (set->use ||
5382             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5383              atomic_read(&set->nelems) > 0)) {
5384                 NL_SET_BAD_ATTR(extack, attr);
5385                 return -EBUSY;
5386         }
5387
5388         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5389
5390         return nft_delset(&ctx, set);
5391 }
5392
5393 static int nft_validate_register_store(const struct nft_ctx *ctx,
5394                                        enum nft_registers reg,
5395                                        const struct nft_data *data,
5396                                        enum nft_data_types type,
5397                                        unsigned int len);
5398
5399 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5400                                      struct nft_set *set,
5401                                      struct nft_elem_priv *elem_priv)
5402 {
5403         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5404         enum nft_registers dreg;
5405
5406         dreg = nft_type_to_reg(set->dtype);
5407         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5408                                            set->dtype == NFT_DATA_VERDICT ?
5409                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5410                                            set->dlen);
5411 }
5412
5413 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5414                                         struct nft_set *set,
5415                                         const struct nft_set_iter *iter,
5416                                         struct nft_elem_priv *elem_priv)
5417 {
5418         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5419
5420         if (!nft_set_elem_active(ext, iter->genmask))
5421                 return 0;
5422
5423         return nft_setelem_data_validate(ctx, set, elem_priv);
5424 }
5425
5426 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5427                                        struct nft_set *set)
5428 {
5429         u8 genmask = nft_genmask_next(ctx->net);
5430         struct nft_set_elem_catchall *catchall;
5431         struct nft_set_ext *ext;
5432         int ret = 0;
5433
5434         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5435                 ext = nft_set_elem_ext(set, catchall->elem);
5436                 if (!nft_set_elem_active(ext, genmask))
5437                         continue;
5438
5439                 ret = nft_setelem_data_validate(ctx, set, catchall->elem);
5440                 if (ret < 0)
5441                         break;
5442         }
5443
5444         return ret;
5445 }
5446
5447 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5448                        struct nft_set_binding *binding)
5449 {
5450         struct nft_set_binding *i;
5451         struct nft_set_iter iter;
5452
5453         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5454                 return -EBUSY;
5455
5456         if (binding->flags & NFT_SET_MAP) {
5457                 /* If the set is already bound to the same chain all
5458                  * jumps are already validated for that chain.
5459                  */
5460                 list_for_each_entry(i, &set->bindings, list) {
5461                         if (i->flags & NFT_SET_MAP &&
5462                             i->chain == binding->chain)
5463                                 goto bind;
5464                 }
5465
5466                 iter.genmask    = nft_genmask_next(ctx->net);
5467                 iter.type       = NFT_ITER_UPDATE;
5468                 iter.skip       = 0;
5469                 iter.count      = 0;
5470                 iter.err        = 0;
5471                 iter.fn         = nf_tables_bind_check_setelem;
5472
5473                 set->ops->walk(ctx, set, &iter);
5474                 if (!iter.err)
5475                         iter.err = nft_set_catchall_bind_check(ctx, set);
5476
5477                 if (iter.err < 0)
5478                         return iter.err;
5479         }
5480 bind:
5481         if (!nft_use_inc(&set->use))
5482                 return -EMFILE;
5483
5484         binding->chain = ctx->chain;
5485         list_add_tail_rcu(&binding->list, &set->bindings);
5486         nft_set_trans_bind(ctx, set);
5487
5488         return 0;
5489 }
5490 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5491
5492 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5493                                  struct nft_set_binding *binding, bool event)
5494 {
5495         list_del_rcu(&binding->list);
5496
5497         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5498                 list_del_rcu(&set->list);
5499                 set->dead = 1;
5500                 if (event)
5501                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5502                                              GFP_KERNEL);
5503         }
5504 }
5505
5506 static void nft_setelem_data_activate(const struct net *net,
5507                                       const struct nft_set *set,
5508                                       struct nft_elem_priv *elem_priv);
5509
5510 static int nft_mapelem_activate(const struct nft_ctx *ctx,
5511                                 struct nft_set *set,
5512                                 const struct nft_set_iter *iter,
5513                                 struct nft_elem_priv *elem_priv)
5514 {
5515         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5516
5517         /* called from abort path, reverse check to undo changes. */
5518         if (nft_set_elem_active(ext, iter->genmask))
5519                 return 0;
5520
5521         nft_clear(ctx->net, ext);
5522         nft_setelem_data_activate(ctx->net, set, elem_priv);
5523
5524         return 0;
5525 }
5526
5527 static void nft_map_catchall_activate(const struct nft_ctx *ctx,
5528                                       struct nft_set *set)
5529 {
5530         u8 genmask = nft_genmask_next(ctx->net);
5531         struct nft_set_elem_catchall *catchall;
5532         struct nft_set_ext *ext;
5533
5534         list_for_each_entry(catchall, &set->catchall_list, list) {
5535                 ext = nft_set_elem_ext(set, catchall->elem);
5536                 if (!nft_set_elem_active(ext, genmask))
5537                         continue;
5538
5539                 nft_clear(ctx->net, ext);
5540                 nft_setelem_data_activate(ctx->net, set, catchall->elem);
5541                 break;
5542         }
5543 }
5544
5545 static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
5546 {
5547         struct nft_set_iter iter = {
5548                 .genmask        = nft_genmask_next(ctx->net),
5549                 .type           = NFT_ITER_UPDATE,
5550                 .fn             = nft_mapelem_activate,
5551         };
5552
5553         set->ops->walk(ctx, set, &iter);
5554         WARN_ON_ONCE(iter.err);
5555
5556         nft_map_catchall_activate(ctx, set);
5557 }
5558
5559 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5560 {
5561         if (nft_set_is_anonymous(set)) {
5562                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5563                         nft_map_activate(ctx, set);
5564
5565                 nft_clear(ctx->net, set);
5566         }
5567
5568         nft_use_inc_restore(&set->use);
5569 }
5570 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5571
5572 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5573                               struct nft_set_binding *binding,
5574                               enum nft_trans_phase phase)
5575 {
5576         switch (phase) {
5577         case NFT_TRANS_PREPARE_ERROR:
5578                 nft_set_trans_unbind(ctx, set);
5579                 if (nft_set_is_anonymous(set))
5580                         nft_deactivate_next(ctx->net, set);
5581                 else
5582                         list_del_rcu(&binding->list);
5583
5584                 nft_use_dec(&set->use);
5585                 break;
5586         case NFT_TRANS_PREPARE:
5587                 if (nft_set_is_anonymous(set)) {
5588                         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5589                                 nft_map_deactivate(ctx, set);
5590
5591                         nft_deactivate_next(ctx->net, set);
5592                 }
5593                 nft_use_dec(&set->use);
5594                 return;
5595         case NFT_TRANS_ABORT:
5596         case NFT_TRANS_RELEASE:
5597                 if (nft_set_is_anonymous(set) &&
5598                     set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5599                         nft_map_deactivate(ctx, set);
5600
5601                 nft_use_dec(&set->use);
5602                 fallthrough;
5603         default:
5604                 nf_tables_unbind_set(ctx, set, binding,
5605                                      phase == NFT_TRANS_COMMIT);
5606         }
5607 }
5608 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5609
5610 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5611 {
5612         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5613                 nft_set_destroy(ctx, set);
5614 }
5615 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5616
5617 const struct nft_set_ext_type nft_set_ext_types[] = {
5618         [NFT_SET_EXT_KEY]               = {
5619                 .align  = __alignof__(u32),
5620         },
5621         [NFT_SET_EXT_DATA]              = {
5622                 .align  = __alignof__(u32),
5623         },
5624         [NFT_SET_EXT_EXPRESSIONS]       = {
5625                 .align  = __alignof__(struct nft_set_elem_expr),
5626         },
5627         [NFT_SET_EXT_OBJREF]            = {
5628                 .len    = sizeof(struct nft_object *),
5629                 .align  = __alignof__(struct nft_object *),
5630         },
5631         [NFT_SET_EXT_FLAGS]             = {
5632                 .len    = sizeof(u8),
5633                 .align  = __alignof__(u8),
5634         },
5635         [NFT_SET_EXT_TIMEOUT]           = {
5636                 .len    = sizeof(u64),
5637                 .align  = __alignof__(u64),
5638         },
5639         [NFT_SET_EXT_EXPIRATION]        = {
5640                 .len    = sizeof(u64),
5641                 .align  = __alignof__(u64),
5642         },
5643         [NFT_SET_EXT_USERDATA]          = {
5644                 .len    = sizeof(struct nft_userdata),
5645                 .align  = __alignof__(struct nft_userdata),
5646         },
5647         [NFT_SET_EXT_KEY_END]           = {
5648                 .align  = __alignof__(u32),
5649         },
5650 };
5651
5652 /*
5653  * Set elements
5654  */
5655
5656 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5657         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5658         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5659         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5660         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5661         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5662         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5663                                             .len = NFT_USERDATA_MAXLEN },
5664         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5665         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5666                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5667         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5668         [NFTA_SET_ELEM_EXPRESSIONS]     = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
5669 };
5670
5671 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5672         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5673                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5674         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5675                                             .len = NFT_SET_MAXNAMELEN - 1 },
5676         [NFTA_SET_ELEM_LIST_ELEMENTS]   = NLA_POLICY_NESTED_ARRAY(nft_set_elem_policy),
5677         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5678 };
5679
5680 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5681                                   const struct nft_set *set,
5682                                   const struct nft_set_ext *ext,
5683                                   bool reset)
5684 {
5685         struct nft_set_elem_expr *elem_expr;
5686         u32 size, num_exprs = 0;
5687         struct nft_expr *expr;
5688         struct nlattr *nest;
5689
5690         elem_expr = nft_set_ext_expr(ext);
5691         nft_setelem_expr_foreach(expr, elem_expr, size)
5692                 num_exprs++;
5693
5694         if (num_exprs == 1) {
5695                 expr = nft_setelem_expr_at(elem_expr, 0);
5696                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, reset) < 0)
5697                         return -1;
5698
5699                 return 0;
5700         } else if (num_exprs > 1) {
5701                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5702                 if (nest == NULL)
5703                         goto nla_put_failure;
5704
5705                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5706                         expr = nft_setelem_expr_at(elem_expr, size);
5707                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
5708                                 goto nla_put_failure;
5709                 }
5710                 nla_nest_end(skb, nest);
5711         }
5712         return 0;
5713
5714 nla_put_failure:
5715         return -1;
5716 }
5717
5718 static int nf_tables_fill_setelem(struct sk_buff *skb,
5719                                   const struct nft_set *set,
5720                                   const struct nft_elem_priv *elem_priv,
5721                                   bool reset)
5722 {
5723         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5724         unsigned char *b = skb_tail_pointer(skb);
5725         struct nlattr *nest;
5726
5727         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5728         if (nest == NULL)
5729                 goto nla_put_failure;
5730
5731         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5732             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5733                           NFT_DATA_VALUE, set->klen) < 0)
5734                 goto nla_put_failure;
5735
5736         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5737             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5738                           NFT_DATA_VALUE, set->klen) < 0)
5739                 goto nla_put_failure;
5740
5741         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5742             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5743                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5744                           set->dlen) < 0)
5745                 goto nla_put_failure;
5746
5747         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5748             nft_set_elem_expr_dump(skb, set, ext, reset))
5749                 goto nla_put_failure;
5750
5751         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5752             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5753                            (*nft_set_ext_obj(ext))->key.name) < 0)
5754                 goto nla_put_failure;
5755
5756         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5757             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5758                          htonl(*nft_set_ext_flags(ext))))
5759                 goto nla_put_failure;
5760
5761         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
5762             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5763                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
5764                          NFTA_SET_ELEM_PAD))
5765                 goto nla_put_failure;
5766
5767         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5768                 u64 expires, now = get_jiffies_64();
5769
5770                 expires = *nft_set_ext_expiration(ext);
5771                 if (time_before64(now, expires))
5772                         expires -= now;
5773                 else
5774                         expires = 0;
5775
5776                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5777                                  nf_jiffies64_to_msecs(expires),
5778                                  NFTA_SET_ELEM_PAD))
5779                         goto nla_put_failure;
5780         }
5781
5782         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5783                 struct nft_userdata *udata;
5784
5785                 udata = nft_set_ext_userdata(ext);
5786                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5787                             udata->len + 1, udata->data))
5788                         goto nla_put_failure;
5789         }
5790
5791         nla_nest_end(skb, nest);
5792         return 0;
5793
5794 nla_put_failure:
5795         nlmsg_trim(skb, b);
5796         return -EMSGSIZE;
5797 }
5798
5799 struct nft_set_dump_args {
5800         const struct netlink_callback   *cb;
5801         struct nft_set_iter             iter;
5802         struct sk_buff                  *skb;
5803         bool                            reset;
5804 };
5805
5806 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5807                                   struct nft_set *set,
5808                                   const struct nft_set_iter *iter,
5809                                   struct nft_elem_priv *elem_priv)
5810 {
5811         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5812         struct nft_set_dump_args *args;
5813
5814         if (!nft_set_elem_active(ext, iter->genmask))
5815                 return 0;
5816
5817         if (nft_set_elem_expired(ext) || nft_set_elem_is_dead(ext))
5818                 return 0;
5819
5820         args = container_of(iter, struct nft_set_dump_args, iter);
5821         return nf_tables_fill_setelem(args->skb, set, elem_priv, args->reset);
5822 }
5823
5824 static void audit_log_nft_set_reset(const struct nft_table *table,
5825                                     unsigned int base_seq,
5826                                     unsigned int nentries)
5827 {
5828         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
5829
5830         audit_log_nfcfg(buf, table->family, nentries,
5831                         AUDIT_NFT_OP_SETELEM_RESET, GFP_ATOMIC);
5832         kfree(buf);
5833 }
5834
5835 struct nft_set_dump_ctx {
5836         const struct nft_set    *set;
5837         struct nft_ctx          ctx;
5838         bool                    reset;
5839 };
5840
5841 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5842                                  const struct nft_set *set, bool reset,
5843                                  unsigned int base_seq)
5844 {
5845         struct nft_set_elem_catchall *catchall;
5846         u8 genmask = nft_genmask_cur(net);
5847         struct nft_set_ext *ext;
5848         int ret = 0;
5849
5850         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5851                 ext = nft_set_elem_ext(set, catchall->elem);
5852                 if (!nft_set_elem_active(ext, genmask) ||
5853                     nft_set_elem_expired(ext))
5854                         continue;
5855
5856                 ret = nf_tables_fill_setelem(skb, set, catchall->elem, reset);
5857                 if (reset && !ret)
5858                         audit_log_nft_set_reset(set->table, base_seq, 1);
5859                 break;
5860         }
5861
5862         return ret;
5863 }
5864
5865 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5866 {
5867         struct nft_set_dump_ctx *dump_ctx = cb->data;
5868         struct net *net = sock_net(skb->sk);
5869         struct nftables_pernet *nft_net;
5870         struct nft_table *table;
5871         struct nft_set *set;
5872         struct nft_set_dump_args args;
5873         bool set_found = false;
5874         struct nlmsghdr *nlh;
5875         struct nlattr *nest;
5876         u32 portid, seq;
5877         int event;
5878
5879         rcu_read_lock();
5880         nft_net = nft_pernet(net);
5881         cb->seq = READ_ONCE(nft_net->base_seq);
5882
5883         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5884                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5885                     dump_ctx->ctx.family != table->family)
5886                         continue;
5887
5888                 if (table != dump_ctx->ctx.table)
5889                         continue;
5890
5891                 list_for_each_entry_rcu(set, &table->sets, list) {
5892                         if (set == dump_ctx->set) {
5893                                 set_found = true;
5894                                 break;
5895                         }
5896                 }
5897                 break;
5898         }
5899
5900         if (!set_found) {
5901                 rcu_read_unlock();
5902                 return -ENOENT;
5903         }
5904
5905         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5906         portid = NETLINK_CB(cb->skb).portid;
5907         seq    = cb->nlh->nlmsg_seq;
5908
5909         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5910                            table->family, NFNETLINK_V0, nft_base_seq(net));
5911         if (!nlh)
5912                 goto nla_put_failure;
5913
5914         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5915                 goto nla_put_failure;
5916         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5917                 goto nla_put_failure;
5918
5919         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5920         if (nest == NULL)
5921                 goto nla_put_failure;
5922
5923         args.cb                 = cb;
5924         args.skb                = skb;
5925         args.reset              = dump_ctx->reset;
5926         args.iter.genmask       = nft_genmask_cur(net);
5927         args.iter.type          = NFT_ITER_READ;
5928         args.iter.skip          = cb->args[0];
5929         args.iter.count         = 0;
5930         args.iter.err           = 0;
5931         args.iter.fn            = nf_tables_dump_setelem;
5932         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5933
5934         if (!args.iter.err && args.iter.count == cb->args[0])
5935                 args.iter.err = nft_set_catchall_dump(net, skb, set,
5936                                                       dump_ctx->reset, cb->seq);
5937         nla_nest_end(skb, nest);
5938         nlmsg_end(skb, nlh);
5939
5940         rcu_read_unlock();
5941
5942         if (args.iter.err && args.iter.err != -EMSGSIZE)
5943                 return args.iter.err;
5944         if (args.iter.count == cb->args[0])
5945                 return 0;
5946
5947         cb->args[0] = args.iter.count;
5948         return skb->len;
5949
5950 nla_put_failure:
5951         rcu_read_unlock();
5952         return -ENOSPC;
5953 }
5954
5955 static int nf_tables_dumpreset_set(struct sk_buff *skb,
5956                                    struct netlink_callback *cb)
5957 {
5958         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
5959         struct nft_set_dump_ctx *dump_ctx = cb->data;
5960         int ret, skip = cb->args[0];
5961
5962         mutex_lock(&nft_net->commit_mutex);
5963
5964         ret = nf_tables_dump_set(skb, cb);
5965
5966         if (cb->args[0] > skip)
5967                 audit_log_nft_set_reset(dump_ctx->ctx.table, cb->seq,
5968                                         cb->args[0] - skip);
5969
5970         mutex_unlock(&nft_net->commit_mutex);
5971
5972         return ret;
5973 }
5974
5975 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5976 {
5977         struct nft_set_dump_ctx *dump_ctx = cb->data;
5978
5979         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5980
5981         return cb->data ? 0 : -ENOMEM;
5982 }
5983
5984 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5985 {
5986         kfree(cb->data);
5987         return 0;
5988 }
5989
5990 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5991                                        const struct nft_ctx *ctx, u32 seq,
5992                                        u32 portid, int event, u16 flags,
5993                                        const struct nft_set *set,
5994                                        const struct nft_elem_priv *elem_priv,
5995                                        bool reset)
5996 {
5997         struct nlmsghdr *nlh;
5998         struct nlattr *nest;
5999         int err;
6000
6001         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
6002         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
6003                            NFNETLINK_V0, nft_base_seq(ctx->net));
6004         if (!nlh)
6005                 goto nla_put_failure;
6006
6007         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
6008                 goto nla_put_failure;
6009         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
6010                 goto nla_put_failure;
6011
6012         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
6013         if (nest == NULL)
6014                 goto nla_put_failure;
6015
6016         err = nf_tables_fill_setelem(skb, set, elem_priv, reset);
6017         if (err < 0)
6018                 goto nla_put_failure;
6019
6020         nla_nest_end(skb, nest);
6021
6022         nlmsg_end(skb, nlh);
6023         return 0;
6024
6025 nla_put_failure:
6026         nlmsg_trim(skb, nlh);
6027         return -1;
6028 }
6029
6030 static int nft_setelem_parse_flags(const struct nft_set *set,
6031                                    const struct nlattr *attr, u32 *flags)
6032 {
6033         if (attr == NULL)
6034                 return 0;
6035
6036         *flags = ntohl(nla_get_be32(attr));
6037         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
6038                 return -EOPNOTSUPP;
6039         if (!(set->flags & NFT_SET_INTERVAL) &&
6040             *flags & NFT_SET_ELEM_INTERVAL_END)
6041                 return -EINVAL;
6042         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
6043             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
6044                 return -EINVAL;
6045
6046         return 0;
6047 }
6048
6049 static int nft_setelem_parse_key(struct nft_ctx *ctx, const struct nft_set *set,
6050                                  struct nft_data *key, struct nlattr *attr)
6051 {
6052         struct nft_data_desc desc = {
6053                 .type   = NFT_DATA_VALUE,
6054                 .size   = NFT_DATA_VALUE_MAXLEN,
6055                 .len    = set->klen,
6056         };
6057
6058         return nft_data_init(ctx, key, &desc, attr);
6059 }
6060
6061 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
6062                                   struct nft_data_desc *desc,
6063                                   struct nft_data *data,
6064                                   struct nlattr *attr)
6065 {
6066         u32 dtype;
6067
6068         if (set->dtype == NFT_DATA_VERDICT)
6069                 dtype = NFT_DATA_VERDICT;
6070         else
6071                 dtype = NFT_DATA_VALUE;
6072
6073         desc->type = dtype;
6074         desc->size = NFT_DATA_VALUE_MAXLEN;
6075         desc->len = set->dlen;
6076         desc->flags = NFT_DATA_DESC_SETELEM;
6077
6078         return nft_data_init(ctx, data, desc, attr);
6079 }
6080
6081 static void *nft_setelem_catchall_get(const struct net *net,
6082                                       const struct nft_set *set)
6083 {
6084         struct nft_set_elem_catchall *catchall;
6085         u8 genmask = nft_genmask_cur(net);
6086         struct nft_set_ext *ext;
6087         void *priv = NULL;
6088
6089         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6090                 ext = nft_set_elem_ext(set, catchall->elem);
6091                 if (!nft_set_elem_active(ext, genmask) ||
6092                     nft_set_elem_expired(ext))
6093                         continue;
6094
6095                 priv = catchall->elem;
6096                 break;
6097         }
6098
6099         return priv;
6100 }
6101
6102 static int nft_setelem_get(struct nft_ctx *ctx, const struct nft_set *set,
6103                            struct nft_set_elem *elem, u32 flags)
6104 {
6105         void *priv;
6106
6107         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
6108                 priv = set->ops->get(ctx->net, set, elem, flags);
6109                 if (IS_ERR(priv))
6110                         return PTR_ERR(priv);
6111         } else {
6112                 priv = nft_setelem_catchall_get(ctx->net, set);
6113                 if (!priv)
6114                         return -ENOENT;
6115         }
6116         elem->priv = priv;
6117
6118         return 0;
6119 }
6120
6121 static int nft_get_set_elem(struct nft_ctx *ctx, const struct nft_set *set,
6122                             const struct nlattr *attr, bool reset)
6123 {
6124         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6125         struct nft_set_elem elem;
6126         struct sk_buff *skb;
6127         uint32_t flags = 0;
6128         int err;
6129
6130         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6131                                           nft_set_elem_policy, NULL);
6132         if (err < 0)
6133                 return err;
6134
6135         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6136         if (err < 0)
6137                 return err;
6138
6139         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6140                 return -EINVAL;
6141
6142         if (nla[NFTA_SET_ELEM_KEY]) {
6143                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6144                                             nla[NFTA_SET_ELEM_KEY]);
6145                 if (err < 0)
6146                         return err;
6147         }
6148
6149         if (nla[NFTA_SET_ELEM_KEY_END]) {
6150                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6151                                             nla[NFTA_SET_ELEM_KEY_END]);
6152                 if (err < 0)
6153                         return err;
6154         }
6155
6156         err = nft_setelem_get(ctx, set, &elem, flags);
6157         if (err < 0)
6158                 return err;
6159
6160         err = -ENOMEM;
6161         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
6162         if (skb == NULL)
6163                 return err;
6164
6165         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
6166                                           NFT_MSG_NEWSETELEM, 0, set, elem.priv,
6167                                           reset);
6168         if (err < 0)
6169                 goto err_fill_setelem;
6170
6171         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
6172
6173 err_fill_setelem:
6174         kfree_skb(skb);
6175         return err;
6176 }
6177
6178 static int nft_set_dump_ctx_init(struct nft_set_dump_ctx *dump_ctx,
6179                                  const struct sk_buff *skb,
6180                                  const struct nfnl_info *info,
6181                                  const struct nlattr * const nla[],
6182                                  bool reset)
6183 {
6184         struct netlink_ext_ack *extack = info->extack;
6185         u8 genmask = nft_genmask_cur(info->net);
6186         u8 family = info->nfmsg->nfgen_family;
6187         struct net *net = info->net;
6188         struct nft_table *table;
6189         struct nft_set *set;
6190
6191         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6192                                  genmask, 0);
6193         if (IS_ERR(table)) {
6194                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6195                 return PTR_ERR(table);
6196         }
6197
6198         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6199         if (IS_ERR(set)) {
6200                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
6201                 return PTR_ERR(set);
6202         }
6203
6204         nft_ctx_init(&dump_ctx->ctx, net, skb,
6205                      info->nlh, family, table, NULL, nla);
6206         dump_ctx->set = set;
6207         dump_ctx->reset = reset;
6208         return 0;
6209 }
6210
6211 /* called with rcu_read_lock held */
6212 static int nf_tables_getsetelem(struct sk_buff *skb,
6213                                 const struct nfnl_info *info,
6214                                 const struct nlattr * const nla[])
6215 {
6216         struct netlink_ext_ack *extack = info->extack;
6217         struct nft_set_dump_ctx dump_ctx;
6218         struct nlattr *attr;
6219         int rem, err = 0;
6220
6221         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6222                 struct netlink_dump_control c = {
6223                         .start = nf_tables_dump_set_start,
6224                         .dump = nf_tables_dump_set,
6225                         .done = nf_tables_dump_set_done,
6226                         .module = THIS_MODULE,
6227                 };
6228
6229                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6230                 if (err)
6231                         return err;
6232
6233                 c.data = &dump_ctx;
6234                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6235         }
6236
6237         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6238                 return -EINVAL;
6239
6240         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6241         if (err)
6242                 return err;
6243
6244         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6245                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, false);
6246                 if (err < 0) {
6247                         NL_SET_BAD_ATTR(extack, attr);
6248                         break;
6249                 }
6250         }
6251
6252         return err;
6253 }
6254
6255 static int nf_tables_getsetelem_reset(struct sk_buff *skb,
6256                                       const struct nfnl_info *info,
6257                                       const struct nlattr * const nla[])
6258 {
6259         struct nftables_pernet *nft_net = nft_pernet(info->net);
6260         struct netlink_ext_ack *extack = info->extack;
6261         struct nft_set_dump_ctx dump_ctx;
6262         int rem, err = 0, nelems = 0;
6263         struct nlattr *attr;
6264
6265         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6266                 struct netlink_dump_control c = {
6267                         .start = nf_tables_dump_set_start,
6268                         .dump = nf_tables_dumpreset_set,
6269                         .done = nf_tables_dump_set_done,
6270                         .module = THIS_MODULE,
6271                 };
6272
6273                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6274                 if (err)
6275                         return err;
6276
6277                 c.data = &dump_ctx;
6278                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6279         }
6280
6281         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6282                 return -EINVAL;
6283
6284         if (!try_module_get(THIS_MODULE))
6285                 return -EINVAL;
6286         rcu_read_unlock();
6287         mutex_lock(&nft_net->commit_mutex);
6288         rcu_read_lock();
6289
6290         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6291         if (err)
6292                 goto out_unlock;
6293
6294         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6295                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, true);
6296                 if (err < 0) {
6297                         NL_SET_BAD_ATTR(extack, attr);
6298                         break;
6299                 }
6300                 nelems++;
6301         }
6302         audit_log_nft_set_reset(dump_ctx.ctx.table, nft_net->base_seq, nelems);
6303
6304 out_unlock:
6305         rcu_read_unlock();
6306         mutex_unlock(&nft_net->commit_mutex);
6307         rcu_read_lock();
6308         module_put(THIS_MODULE);
6309
6310         return err;
6311 }
6312
6313 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
6314                                      const struct nft_set *set,
6315                                      const struct nft_elem_priv *elem_priv,
6316                                      int event)
6317 {
6318         struct nftables_pernet *nft_net;
6319         struct net *net = ctx->net;
6320         u32 portid = ctx->portid;
6321         struct sk_buff *skb;
6322         u16 flags = 0;
6323         int err;
6324
6325         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6326                 return;
6327
6328         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
6329         if (skb == NULL)
6330                 goto err;
6331
6332         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
6333                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
6334
6335         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
6336                                           set, elem_priv, false);
6337         if (err < 0) {
6338                 kfree_skb(skb);
6339                 goto err;
6340         }
6341
6342         nft_net = nft_pernet(net);
6343         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
6344         return;
6345 err:
6346         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6347 }
6348
6349 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
6350                                               int msg_type,
6351                                               struct nft_set *set)
6352 {
6353         struct nft_trans *trans;
6354
6355         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
6356         if (trans == NULL)
6357                 return NULL;
6358
6359         nft_trans_elem_set(trans) = set;
6360         return trans;
6361 }
6362
6363 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
6364                                          const struct nft_set *set,
6365                                          const struct nlattr *attr)
6366 {
6367         struct nft_expr *expr;
6368         int err;
6369
6370         expr = nft_expr_init(ctx, attr);
6371         if (IS_ERR(expr))
6372                 return expr;
6373
6374         err = -EOPNOTSUPP;
6375         if (expr->ops->type->flags & NFT_EXPR_GC) {
6376                 if (set->flags & NFT_SET_TIMEOUT)
6377                         goto err_set_elem_expr;
6378                 if (!set->ops->gc_init)
6379                         goto err_set_elem_expr;
6380                 set->ops->gc_init(set);
6381         }
6382
6383         return expr;
6384
6385 err_set_elem_expr:
6386         nft_expr_destroy(ctx, expr);
6387         return ERR_PTR(err);
6388 }
6389
6390 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
6391 {
6392         len += nft_set_ext_types[id].len;
6393         if (len > tmpl->ext_len[id] ||
6394             len > U8_MAX)
6395                 return -1;
6396
6397         return 0;
6398 }
6399
6400 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
6401                               void *to, const void *from, u32 len)
6402 {
6403         if (nft_set_ext_check(tmpl, id, len) < 0)
6404                 return -1;
6405
6406         memcpy(to, from, len);
6407
6408         return 0;
6409 }
6410
6411 struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set,
6412                                         const struct nft_set_ext_tmpl *tmpl,
6413                                         const u32 *key, const u32 *key_end,
6414                                         const u32 *data,
6415                                         u64 timeout, u64 expiration, gfp_t gfp)
6416 {
6417         struct nft_set_ext *ext;
6418         void *elem;
6419
6420         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
6421         if (elem == NULL)
6422                 return ERR_PTR(-ENOMEM);
6423
6424         ext = nft_set_elem_ext(set, elem);
6425         nft_set_ext_init(ext, tmpl);
6426
6427         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
6428             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
6429                                nft_set_ext_key(ext), key, set->klen) < 0)
6430                 goto err_ext_check;
6431
6432         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
6433             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
6434                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
6435                 goto err_ext_check;
6436
6437         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6438             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
6439                                nft_set_ext_data(ext), data, set->dlen) < 0)
6440                 goto err_ext_check;
6441
6442         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
6443                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
6444                 if (expiration == 0)
6445                         *nft_set_ext_expiration(ext) += timeout;
6446         }
6447         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
6448                 *nft_set_ext_timeout(ext) = timeout;
6449
6450         return elem;
6451
6452 err_ext_check:
6453         kfree(elem);
6454
6455         return ERR_PTR(-EINVAL);
6456 }
6457
6458 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6459                                         struct nft_expr *expr)
6460 {
6461         if (expr->ops->destroy_clone) {
6462                 expr->ops->destroy_clone(ctx, expr);
6463                 module_put(expr->ops->type->owner);
6464         } else {
6465                 nf_tables_expr_destroy(ctx, expr);
6466         }
6467 }
6468
6469 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6470                                       struct nft_set_elem_expr *elem_expr)
6471 {
6472         struct nft_expr *expr;
6473         u32 size;
6474
6475         nft_setelem_expr_foreach(expr, elem_expr, size)
6476                 __nft_set_elem_expr_destroy(ctx, expr);
6477 }
6478
6479 /* Drop references and destroy. Called from gc, dynset and abort path. */
6480 void nft_set_elem_destroy(const struct nft_set *set,
6481                           const struct nft_elem_priv *elem_priv,
6482                           bool destroy_expr)
6483 {
6484         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6485         struct nft_ctx ctx = {
6486                 .net    = read_pnet(&set->net),
6487                 .family = set->table->family,
6488         };
6489
6490         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
6491         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6492                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6493         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6494                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
6495         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6496                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6497
6498         kfree(elem_priv);
6499 }
6500 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
6501
6502 /* Destroy element. References have been already dropped in the preparation
6503  * path via nft_setelem_data_deactivate().
6504  */
6505 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6506                                 const struct nft_set *set,
6507                                 const struct nft_elem_priv *elem_priv)
6508 {
6509         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6510
6511         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6512                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6513
6514         kfree(elem_priv);
6515 }
6516
6517 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6518                             struct nft_expr *expr_array[])
6519 {
6520         struct nft_expr *expr;
6521         int err, i, k;
6522
6523         for (i = 0; i < set->num_exprs; i++) {
6524                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6525                 if (!expr)
6526                         goto err_expr;
6527
6528                 err = nft_expr_clone(expr, set->exprs[i]);
6529                 if (err < 0) {
6530                         kfree(expr);
6531                         goto err_expr;
6532                 }
6533                 expr_array[i] = expr;
6534         }
6535
6536         return 0;
6537
6538 err_expr:
6539         for (k = i - 1; k >= 0; k--)
6540                 nft_expr_destroy(ctx, expr_array[k]);
6541
6542         return -ENOMEM;
6543 }
6544
6545 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6546                                    const struct nft_set_ext_tmpl *tmpl,
6547                                    const struct nft_set_ext *ext,
6548                                    struct nft_expr *expr_array[],
6549                                    u32 num_exprs)
6550 {
6551         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6552         u32 len = sizeof(struct nft_set_elem_expr);
6553         struct nft_expr *expr;
6554         int i, err;
6555
6556         if (num_exprs == 0)
6557                 return 0;
6558
6559         for (i = 0; i < num_exprs; i++)
6560                 len += expr_array[i]->ops->size;
6561
6562         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6563                 return -EINVAL;
6564
6565         for (i = 0; i < num_exprs; i++) {
6566                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6567                 err = nft_expr_clone(expr, expr_array[i]);
6568                 if (err < 0)
6569                         goto err_elem_expr_setup;
6570
6571                 elem_expr->size += expr_array[i]->ops->size;
6572                 nft_expr_destroy(ctx, expr_array[i]);
6573                 expr_array[i] = NULL;
6574         }
6575
6576         return 0;
6577
6578 err_elem_expr_setup:
6579         for (; i < num_exprs; i++) {
6580                 nft_expr_destroy(ctx, expr_array[i]);
6581                 expr_array[i] = NULL;
6582         }
6583
6584         return -ENOMEM;
6585 }
6586
6587 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6588                                             const struct nft_set *set)
6589 {
6590         struct nft_set_elem_catchall *catchall;
6591         u8 genmask = nft_genmask_cur(net);
6592         struct nft_set_ext *ext;
6593
6594         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6595                 ext = nft_set_elem_ext(set, catchall->elem);
6596                 if (nft_set_elem_active(ext, genmask) &&
6597                     !nft_set_elem_expired(ext) &&
6598                     !nft_set_elem_is_dead(ext))
6599                         return ext;
6600         }
6601
6602         return NULL;
6603 }
6604 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6605
6606 static int nft_setelem_catchall_insert(const struct net *net,
6607                                        struct nft_set *set,
6608                                        const struct nft_set_elem *elem,
6609                                        struct nft_elem_priv **priv)
6610 {
6611         struct nft_set_elem_catchall *catchall;
6612         u8 genmask = nft_genmask_next(net);
6613         struct nft_set_ext *ext;
6614
6615         list_for_each_entry(catchall, &set->catchall_list, list) {
6616                 ext = nft_set_elem_ext(set, catchall->elem);
6617                 if (nft_set_elem_active(ext, genmask)) {
6618                         *priv = catchall->elem;
6619                         return -EEXIST;
6620                 }
6621         }
6622
6623         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6624         if (!catchall)
6625                 return -ENOMEM;
6626
6627         catchall->elem = elem->priv;
6628         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6629
6630         return 0;
6631 }
6632
6633 static int nft_setelem_insert(const struct net *net,
6634                               struct nft_set *set,
6635                               const struct nft_set_elem *elem,
6636                               struct nft_elem_priv **elem_priv,
6637                               unsigned int flags)
6638 {
6639         int ret;
6640
6641         if (flags & NFT_SET_ELEM_CATCHALL)
6642                 ret = nft_setelem_catchall_insert(net, set, elem, elem_priv);
6643         else
6644                 ret = set->ops->insert(net, set, elem, elem_priv);
6645
6646         return ret;
6647 }
6648
6649 static bool nft_setelem_is_catchall(const struct nft_set *set,
6650                                     const struct nft_elem_priv *elem_priv)
6651 {
6652         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6653
6654         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6655             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6656                 return true;
6657
6658         return false;
6659 }
6660
6661 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6662                                  struct nft_elem_priv *elem_priv)
6663 {
6664         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6665
6666         if (nft_setelem_is_catchall(set, elem_priv)) {
6667                 nft_clear(net, ext);
6668         } else {
6669                 set->ops->activate(net, set, elem_priv);
6670         }
6671 }
6672
6673 static int nft_setelem_catchall_deactivate(const struct net *net,
6674                                            struct nft_set *set,
6675                                            struct nft_set_elem *elem)
6676 {
6677         struct nft_set_elem_catchall *catchall;
6678         struct nft_set_ext *ext;
6679
6680         list_for_each_entry(catchall, &set->catchall_list, list) {
6681                 ext = nft_set_elem_ext(set, catchall->elem);
6682                 if (!nft_is_active_next(net, ext))
6683                         continue;
6684
6685                 kfree(elem->priv);
6686                 elem->priv = catchall->elem;
6687                 nft_set_elem_change_active(net, set, ext);
6688                 return 0;
6689         }
6690
6691         return -ENOENT;
6692 }
6693
6694 static int __nft_setelem_deactivate(const struct net *net,
6695                                     struct nft_set *set,
6696                                     struct nft_set_elem *elem)
6697 {
6698         void *priv;
6699
6700         priv = set->ops->deactivate(net, set, elem);
6701         if (!priv)
6702                 return -ENOENT;
6703
6704         kfree(elem->priv);
6705         elem->priv = priv;
6706         set->ndeact++;
6707
6708         return 0;
6709 }
6710
6711 static int nft_setelem_deactivate(const struct net *net,
6712                                   struct nft_set *set,
6713                                   struct nft_set_elem *elem, u32 flags)
6714 {
6715         int ret;
6716
6717         if (flags & NFT_SET_ELEM_CATCHALL)
6718                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6719         else
6720                 ret = __nft_setelem_deactivate(net, set, elem);
6721
6722         return ret;
6723 }
6724
6725 static void nft_setelem_catchall_destroy(struct nft_set_elem_catchall *catchall)
6726 {
6727         list_del_rcu(&catchall->list);
6728         kfree_rcu(catchall, rcu);
6729 }
6730
6731 static void nft_setelem_catchall_remove(const struct net *net,
6732                                         const struct nft_set *set,
6733                                         struct nft_elem_priv *elem_priv)
6734 {
6735         struct nft_set_elem_catchall *catchall, *next;
6736
6737         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6738                 if (catchall->elem == elem_priv) {
6739                         nft_setelem_catchall_destroy(catchall);
6740                         break;
6741                 }
6742         }
6743 }
6744
6745 static void nft_setelem_remove(const struct net *net,
6746                                const struct nft_set *set,
6747                                struct nft_elem_priv *elem_priv)
6748 {
6749         if (nft_setelem_is_catchall(set, elem_priv))
6750                 nft_setelem_catchall_remove(net, set, elem_priv);
6751         else
6752                 set->ops->remove(net, set, elem_priv);
6753 }
6754
6755 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6756                                       struct nlattr **nla, u32 flags)
6757 {
6758         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6759                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6760                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6761                         return false;
6762
6763                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6764                     flags & NFT_SET_ELEM_CATCHALL)
6765                         return false;
6766         } else {
6767                 if (nla[NFTA_SET_ELEM_KEY_END])
6768                         return false;
6769         }
6770
6771         return true;
6772 }
6773
6774 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6775                             const struct nlattr *attr, u32 nlmsg_flags)
6776 {
6777         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6778         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6779         u8 genmask = nft_genmask_next(ctx->net);
6780         u32 flags = 0, size = 0, num_exprs = 0;
6781         struct nft_set_ext_tmpl tmpl;
6782         struct nft_set_ext *ext, *ext2;
6783         struct nft_set_elem elem;
6784         struct nft_set_binding *binding;
6785         struct nft_elem_priv *elem_priv;
6786         struct nft_object *obj = NULL;
6787         struct nft_userdata *udata;
6788         struct nft_data_desc desc;
6789         enum nft_registers dreg;
6790         struct nft_trans *trans;
6791         u64 expiration;
6792         u64 timeout;
6793         int err, i;
6794         u8 ulen;
6795
6796         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6797                                           nft_set_elem_policy, NULL);
6798         if (err < 0)
6799                 return err;
6800
6801         nft_set_ext_prepare(&tmpl);
6802
6803         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6804         if (err < 0)
6805                 return err;
6806
6807         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6808             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6809                 return -EINVAL;
6810
6811         if (flags != 0) {
6812                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6813                 if (err < 0)
6814                         return err;
6815         }
6816
6817         if (set->flags & NFT_SET_MAP) {
6818                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6819                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6820                         return -EINVAL;
6821         } else {
6822                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6823                         return -EINVAL;
6824         }
6825
6826         if (set->flags & NFT_SET_OBJECT) {
6827                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6828                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6829                         return -EINVAL;
6830         } else {
6831                 if (nla[NFTA_SET_ELEM_OBJREF])
6832                         return -EINVAL;
6833         }
6834
6835         if (!nft_setelem_valid_key_end(set, nla, flags))
6836                 return -EINVAL;
6837
6838         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6839              (nla[NFTA_SET_ELEM_DATA] ||
6840               nla[NFTA_SET_ELEM_OBJREF] ||
6841               nla[NFTA_SET_ELEM_TIMEOUT] ||
6842               nla[NFTA_SET_ELEM_EXPIRATION] ||
6843               nla[NFTA_SET_ELEM_USERDATA] ||
6844               nla[NFTA_SET_ELEM_EXPR] ||
6845               nla[NFTA_SET_ELEM_KEY_END] ||
6846               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6847                 return -EINVAL;
6848
6849         timeout = 0;
6850         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6851                 if (!(set->flags & NFT_SET_TIMEOUT))
6852                         return -EINVAL;
6853                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6854                                             &timeout);
6855                 if (err)
6856                         return err;
6857         } else if (set->flags & NFT_SET_TIMEOUT &&
6858                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6859                 timeout = READ_ONCE(set->timeout);
6860         }
6861
6862         expiration = 0;
6863         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6864                 if (!(set->flags & NFT_SET_TIMEOUT))
6865                         return -EINVAL;
6866                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6867                                             &expiration);
6868                 if (err)
6869                         return err;
6870         }
6871
6872         if (nla[NFTA_SET_ELEM_EXPR]) {
6873                 struct nft_expr *expr;
6874
6875                 if (set->num_exprs && set->num_exprs != 1)
6876                         return -EOPNOTSUPP;
6877
6878                 expr = nft_set_elem_expr_alloc(ctx, set,
6879                                                nla[NFTA_SET_ELEM_EXPR]);
6880                 if (IS_ERR(expr))
6881                         return PTR_ERR(expr);
6882
6883                 expr_array[0] = expr;
6884                 num_exprs = 1;
6885
6886                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6887                         err = -EOPNOTSUPP;
6888                         goto err_set_elem_expr;
6889                 }
6890         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6891                 struct nft_expr *expr;
6892                 struct nlattr *tmp;
6893                 int left;
6894
6895                 i = 0;
6896                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6897                         if (i == NFT_SET_EXPR_MAX ||
6898                             (set->num_exprs && set->num_exprs == i)) {
6899                                 err = -E2BIG;
6900                                 goto err_set_elem_expr;
6901                         }
6902                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6903                                 err = -EINVAL;
6904                                 goto err_set_elem_expr;
6905                         }
6906                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6907                         if (IS_ERR(expr)) {
6908                                 err = PTR_ERR(expr);
6909                                 goto err_set_elem_expr;
6910                         }
6911                         expr_array[i] = expr;
6912                         num_exprs++;
6913
6914                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6915                                 err = -EOPNOTSUPP;
6916                                 goto err_set_elem_expr;
6917                         }
6918                         i++;
6919                 }
6920                 if (set->num_exprs && set->num_exprs != i) {
6921                         err = -EOPNOTSUPP;
6922                         goto err_set_elem_expr;
6923                 }
6924         } else if (set->num_exprs > 0 &&
6925                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6926                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6927                 if (err < 0)
6928                         goto err_set_elem_expr_clone;
6929
6930                 num_exprs = set->num_exprs;
6931         }
6932
6933         if (nla[NFTA_SET_ELEM_KEY]) {
6934                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6935                                             nla[NFTA_SET_ELEM_KEY]);
6936                 if (err < 0)
6937                         goto err_set_elem_expr;
6938
6939                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6940                 if (err < 0)
6941                         goto err_parse_key;
6942         }
6943
6944         if (nla[NFTA_SET_ELEM_KEY_END]) {
6945                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6946                                             nla[NFTA_SET_ELEM_KEY_END]);
6947                 if (err < 0)
6948                         goto err_parse_key;
6949
6950                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6951                 if (err < 0)
6952                         goto err_parse_key_end;
6953         }
6954
6955         if (timeout > 0) {
6956                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6957                 if (err < 0)
6958                         goto err_parse_key_end;
6959
6960                 if (timeout != READ_ONCE(set->timeout)) {
6961                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6962                         if (err < 0)
6963                                 goto err_parse_key_end;
6964                 }
6965         }
6966
6967         if (num_exprs) {
6968                 for (i = 0; i < num_exprs; i++)
6969                         size += expr_array[i]->ops->size;
6970
6971                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6972                                              sizeof(struct nft_set_elem_expr) + size);
6973                 if (err < 0)
6974                         goto err_parse_key_end;
6975         }
6976
6977         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6978                 obj = nft_obj_lookup(ctx->net, ctx->table,
6979                                      nla[NFTA_SET_ELEM_OBJREF],
6980                                      set->objtype, genmask);
6981                 if (IS_ERR(obj)) {
6982                         err = PTR_ERR(obj);
6983                         obj = NULL;
6984                         goto err_parse_key_end;
6985                 }
6986
6987                 if (!nft_use_inc(&obj->use)) {
6988                         err = -EMFILE;
6989                         obj = NULL;
6990                         goto err_parse_key_end;
6991                 }
6992
6993                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6994                 if (err < 0)
6995                         goto err_parse_key_end;
6996         }
6997
6998         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6999                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
7000                                              nla[NFTA_SET_ELEM_DATA]);
7001                 if (err < 0)
7002                         goto err_parse_key_end;
7003
7004                 dreg = nft_type_to_reg(set->dtype);
7005                 list_for_each_entry(binding, &set->bindings, list) {
7006                         struct nft_ctx bind_ctx = {
7007                                 .net    = ctx->net,
7008                                 .family = ctx->family,
7009                                 .table  = ctx->table,
7010                                 .chain  = (struct nft_chain *)binding->chain,
7011                         };
7012
7013                         if (!(binding->flags & NFT_SET_MAP))
7014                                 continue;
7015
7016                         err = nft_validate_register_store(&bind_ctx, dreg,
7017                                                           &elem.data.val,
7018                                                           desc.type, desc.len);
7019                         if (err < 0)
7020                                 goto err_parse_data;
7021
7022                         if (desc.type == NFT_DATA_VERDICT &&
7023                             (elem.data.val.verdict.code == NFT_GOTO ||
7024                              elem.data.val.verdict.code == NFT_JUMP))
7025                                 nft_validate_state_update(ctx->table,
7026                                                           NFT_VALIDATE_NEED);
7027                 }
7028
7029                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
7030                 if (err < 0)
7031                         goto err_parse_data;
7032         }
7033
7034         /* The full maximum length of userdata can exceed the maximum
7035          * offset value (U8_MAX) for following extensions, therefor it
7036          * must be the last extension added.
7037          */
7038         ulen = 0;
7039         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
7040                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
7041                 if (ulen > 0) {
7042                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
7043                                                      ulen);
7044                         if (err < 0)
7045                                 goto err_parse_data;
7046                 }
7047         }
7048
7049         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7050                                       elem.key_end.val.data, elem.data.val.data,
7051                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
7052         if (IS_ERR(elem.priv)) {
7053                 err = PTR_ERR(elem.priv);
7054                 goto err_parse_data;
7055         }
7056
7057         ext = nft_set_elem_ext(set, elem.priv);
7058         if (flags)
7059                 *nft_set_ext_flags(ext) = flags;
7060
7061         if (obj)
7062                 *nft_set_ext_obj(ext) = obj;
7063
7064         if (ulen > 0) {
7065                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
7066                         err = -EINVAL;
7067                         goto err_elem_free;
7068                 }
7069                 udata = nft_set_ext_userdata(ext);
7070                 udata->len = ulen - 1;
7071                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
7072         }
7073         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
7074         if (err < 0)
7075                 goto err_elem_free;
7076
7077         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
7078         if (trans == NULL) {
7079                 err = -ENOMEM;
7080                 goto err_elem_free;
7081         }
7082
7083         ext->genmask = nft_genmask_cur(ctx->net);
7084
7085         err = nft_setelem_insert(ctx->net, set, &elem, &elem_priv, flags);
7086         if (err) {
7087                 if (err == -EEXIST) {
7088                         ext2 = nft_set_elem_ext(set, elem_priv);
7089                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
7090                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
7091                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
7092                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
7093                                 goto err_element_clash;
7094                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
7095                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
7096                              memcmp(nft_set_ext_data(ext),
7097                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
7098                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
7099                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
7100                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
7101                                 goto err_element_clash;
7102                         else if (!(nlmsg_flags & NLM_F_EXCL))
7103                                 err = 0;
7104                 } else if (err == -ENOTEMPTY) {
7105                         /* ENOTEMPTY reports overlapping between this element
7106                          * and an existing one.
7107                          */
7108                         err = -EEXIST;
7109                 }
7110                 goto err_element_clash;
7111         }
7112
7113         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
7114                 unsigned int max = set->size ? set->size + set->ndeact : UINT_MAX;
7115
7116                 if (!atomic_add_unless(&set->nelems, 1, max)) {
7117                         err = -ENFILE;
7118                         goto err_set_full;
7119                 }
7120         }
7121
7122         nft_trans_elem_priv(trans) = elem.priv;
7123         nft_trans_commit_list_add_tail(ctx->net, trans);
7124         return 0;
7125
7126 err_set_full:
7127         nft_setelem_remove(ctx->net, set, elem.priv);
7128 err_element_clash:
7129         kfree(trans);
7130 err_elem_free:
7131         nf_tables_set_elem_destroy(ctx, set, elem.priv);
7132 err_parse_data:
7133         if (nla[NFTA_SET_ELEM_DATA] != NULL)
7134                 nft_data_release(&elem.data.val, desc.type);
7135 err_parse_key_end:
7136         if (obj)
7137                 nft_use_dec_restore(&obj->use);
7138
7139         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7140 err_parse_key:
7141         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7142 err_set_elem_expr:
7143         for (i = 0; i < num_exprs && expr_array[i]; i++)
7144                 nft_expr_destroy(ctx, expr_array[i]);
7145 err_set_elem_expr_clone:
7146         return err;
7147 }
7148
7149 static int nf_tables_newsetelem(struct sk_buff *skb,
7150                                 const struct nfnl_info *info,
7151                                 const struct nlattr * const nla[])
7152 {
7153         struct netlink_ext_ack *extack = info->extack;
7154         u8 genmask = nft_genmask_next(info->net);
7155         u8 family = info->nfmsg->nfgen_family;
7156         struct net *net = info->net;
7157         const struct nlattr *attr;
7158         struct nft_table *table;
7159         struct nft_set *set;
7160         struct nft_ctx ctx;
7161         int rem, err;
7162
7163         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
7164                 return -EINVAL;
7165
7166         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7167                                  genmask, NETLINK_CB(skb).portid);
7168         if (IS_ERR(table)) {
7169                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7170                 return PTR_ERR(table);
7171         }
7172
7173         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
7174                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
7175         if (IS_ERR(set)) {
7176                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7177                 return PTR_ERR(set);
7178         }
7179
7180         if (!list_empty(&set->bindings) &&
7181             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
7182                 return -EBUSY;
7183
7184         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7185
7186         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7187                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
7188                 if (err < 0) {
7189                         NL_SET_BAD_ATTR(extack, attr);
7190                         return err;
7191                 }
7192         }
7193
7194         if (table->validate_state == NFT_VALIDATE_DO)
7195                 return nft_table_validate(net, table);
7196
7197         return 0;
7198 }
7199
7200 /**
7201  *      nft_data_hold - hold a nft_data item
7202  *
7203  *      @data: struct nft_data to release
7204  *      @type: type of data
7205  *
7206  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
7207  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
7208  *      NFT_GOTO verdicts. This function must be called on active data objects
7209  *      from the second phase of the commit protocol.
7210  */
7211 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
7212 {
7213         struct nft_chain *chain;
7214
7215         if (type == NFT_DATA_VERDICT) {
7216                 switch (data->verdict.code) {
7217                 case NFT_JUMP:
7218                 case NFT_GOTO:
7219                         chain = data->verdict.chain;
7220                         nft_use_inc_restore(&chain->use);
7221                         break;
7222                 }
7223         }
7224 }
7225
7226 static int nft_setelem_active_next(const struct net *net,
7227                                    const struct nft_set *set,
7228                                    struct nft_elem_priv *elem_priv)
7229 {
7230         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7231         u8 genmask = nft_genmask_next(net);
7232
7233         return nft_set_elem_active(ext, genmask);
7234 }
7235
7236 static void nft_setelem_data_activate(const struct net *net,
7237                                       const struct nft_set *set,
7238                                       struct nft_elem_priv *elem_priv)
7239 {
7240         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7241
7242         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7243                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
7244         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7245                 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
7246 }
7247
7248 void nft_setelem_data_deactivate(const struct net *net,
7249                                  const struct nft_set *set,
7250                                  struct nft_elem_priv *elem_priv)
7251 {
7252         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7253
7254         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7255                 nft_data_release(nft_set_ext_data(ext), set->dtype);
7256         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7257                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
7258 }
7259
7260 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
7261                            const struct nlattr *attr)
7262 {
7263         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
7264         struct nft_set_ext_tmpl tmpl;
7265         struct nft_set_elem elem;
7266         struct nft_set_ext *ext;
7267         struct nft_trans *trans;
7268         u32 flags = 0;
7269         int err;
7270
7271         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
7272                                           nft_set_elem_policy, NULL);
7273         if (err < 0)
7274                 return err;
7275
7276         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
7277         if (err < 0)
7278                 return err;
7279
7280         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
7281                 return -EINVAL;
7282
7283         if (!nft_setelem_valid_key_end(set, nla, flags))
7284                 return -EINVAL;
7285
7286         nft_set_ext_prepare(&tmpl);
7287
7288         if (flags != 0) {
7289                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
7290                 if (err < 0)
7291                         return err;
7292         }
7293
7294         if (nla[NFTA_SET_ELEM_KEY]) {
7295                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
7296                                             nla[NFTA_SET_ELEM_KEY]);
7297                 if (err < 0)
7298                         return err;
7299
7300                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
7301                 if (err < 0)
7302                         goto fail_elem;
7303         }
7304
7305         if (nla[NFTA_SET_ELEM_KEY_END]) {
7306                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
7307                                             nla[NFTA_SET_ELEM_KEY_END]);
7308                 if (err < 0)
7309                         goto fail_elem;
7310
7311                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
7312                 if (err < 0)
7313                         goto fail_elem_key_end;
7314         }
7315
7316         err = -ENOMEM;
7317         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7318                                       elem.key_end.val.data, NULL, 0, 0,
7319                                       GFP_KERNEL_ACCOUNT);
7320         if (IS_ERR(elem.priv)) {
7321                 err = PTR_ERR(elem.priv);
7322                 goto fail_elem_key_end;
7323         }
7324
7325         ext = nft_set_elem_ext(set, elem.priv);
7326         if (flags)
7327                 *nft_set_ext_flags(ext) = flags;
7328
7329         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
7330         if (trans == NULL)
7331                 goto fail_trans;
7332
7333         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
7334         if (err < 0)
7335                 goto fail_ops;
7336
7337         nft_setelem_data_deactivate(ctx->net, set, elem.priv);
7338
7339         nft_trans_elem_priv(trans) = elem.priv;
7340         nft_trans_commit_list_add_tail(ctx->net, trans);
7341         return 0;
7342
7343 fail_ops:
7344         kfree(trans);
7345 fail_trans:
7346         kfree(elem.priv);
7347 fail_elem_key_end:
7348         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7349 fail_elem:
7350         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7351         return err;
7352 }
7353
7354 static int nft_setelem_flush(const struct nft_ctx *ctx,
7355                              struct nft_set *set,
7356                              const struct nft_set_iter *iter,
7357                              struct nft_elem_priv *elem_priv)
7358 {
7359         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7360         struct nft_trans *trans;
7361
7362         if (!nft_set_elem_active(ext, iter->genmask))
7363                 return 0;
7364
7365         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7366                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
7367         if (!trans)
7368                 return -ENOMEM;
7369
7370         set->ops->flush(ctx->net, set, elem_priv);
7371         set->ndeact++;
7372
7373         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7374         nft_trans_elem_set(trans) = set;
7375         nft_trans_elem_priv(trans) = elem_priv;
7376         nft_trans_commit_list_add_tail(ctx->net, trans);
7377
7378         return 0;
7379 }
7380
7381 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
7382                                     struct nft_set *set,
7383                                     struct nft_elem_priv *elem_priv)
7384 {
7385         struct nft_trans *trans;
7386
7387         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7388                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
7389         if (!trans)
7390                 return -ENOMEM;
7391
7392         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7393         nft_trans_elem_set(trans) = set;
7394         nft_trans_elem_priv(trans) = elem_priv;
7395         nft_trans_commit_list_add_tail(ctx->net, trans);
7396
7397         return 0;
7398 }
7399
7400 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
7401                                   struct nft_set *set)
7402 {
7403         u8 genmask = nft_genmask_next(ctx->net);
7404         struct nft_set_elem_catchall *catchall;
7405         struct nft_set_ext *ext;
7406         int ret = 0;
7407
7408         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
7409                 ext = nft_set_elem_ext(set, catchall->elem);
7410                 if (!nft_set_elem_active(ext, genmask))
7411                         continue;
7412
7413                 ret = __nft_set_catchall_flush(ctx, set, catchall->elem);
7414                 if (ret < 0)
7415                         break;
7416                 nft_set_elem_change_active(ctx->net, set, ext);
7417         }
7418
7419         return ret;
7420 }
7421
7422 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
7423 {
7424         struct nft_set_iter iter = {
7425                 .genmask        = genmask,
7426                 .type           = NFT_ITER_UPDATE,
7427                 .fn             = nft_setelem_flush,
7428         };
7429
7430         set->ops->walk(ctx, set, &iter);
7431         if (!iter.err)
7432                 iter.err = nft_set_catchall_flush(ctx, set);
7433
7434         return iter.err;
7435 }
7436
7437 static int nf_tables_delsetelem(struct sk_buff *skb,
7438                                 const struct nfnl_info *info,
7439                                 const struct nlattr * const nla[])
7440 {
7441         struct netlink_ext_ack *extack = info->extack;
7442         u8 genmask = nft_genmask_next(info->net);
7443         u8 family = info->nfmsg->nfgen_family;
7444         struct net *net = info->net;
7445         const struct nlattr *attr;
7446         struct nft_table *table;
7447         struct nft_set *set;
7448         struct nft_ctx ctx;
7449         int rem, err = 0;
7450
7451         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7452                                  genmask, NETLINK_CB(skb).portid);
7453         if (IS_ERR(table)) {
7454                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7455                 return PTR_ERR(table);
7456         }
7457
7458         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
7459         if (IS_ERR(set)) {
7460                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7461                 return PTR_ERR(set);
7462         }
7463
7464         if (nft_set_is_anonymous(set))
7465                 return -EOPNOTSUPP;
7466
7467         if (!list_empty(&set->bindings) && (set->flags & NFT_SET_CONSTANT))
7468                 return -EBUSY;
7469
7470         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7471
7472         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
7473                 return nft_set_flush(&ctx, set, genmask);
7474
7475         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7476                 err = nft_del_setelem(&ctx, set, attr);
7477                 if (err == -ENOENT &&
7478                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
7479                         continue;
7480
7481                 if (err < 0) {
7482                         NL_SET_BAD_ATTR(extack, attr);
7483                         return err;
7484                 }
7485         }
7486
7487         return 0;
7488 }
7489
7490 /*
7491  * Stateful objects
7492  */
7493
7494 /**
7495  *      nft_register_obj- register nf_tables stateful object type
7496  *      @obj_type: object type
7497  *
7498  *      Registers the object type for use with nf_tables. Returns zero on
7499  *      success or a negative errno code otherwise.
7500  */
7501 int nft_register_obj(struct nft_object_type *obj_type)
7502 {
7503         if (obj_type->type == NFT_OBJECT_UNSPEC)
7504                 return -EINVAL;
7505
7506         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7507         list_add_rcu(&obj_type->list, &nf_tables_objects);
7508         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7509         return 0;
7510 }
7511 EXPORT_SYMBOL_GPL(nft_register_obj);
7512
7513 /**
7514  *      nft_unregister_obj - unregister nf_tables object type
7515  *      @obj_type: object type
7516  *
7517  *      Unregisters the object type for use with nf_tables.
7518  */
7519 void nft_unregister_obj(struct nft_object_type *obj_type)
7520 {
7521         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7522         list_del_rcu(&obj_type->list);
7523         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7524 }
7525 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7526
7527 struct nft_object *nft_obj_lookup(const struct net *net,
7528                                   const struct nft_table *table,
7529                                   const struct nlattr *nla, u32 objtype,
7530                                   u8 genmask)
7531 {
7532         struct nft_object_hash_key k = { .table = table };
7533         char search[NFT_OBJ_MAXNAMELEN];
7534         struct rhlist_head *tmp, *list;
7535         struct nft_object *obj;
7536
7537         nla_strscpy(search, nla, sizeof(search));
7538         k.name = search;
7539
7540         WARN_ON_ONCE(!rcu_read_lock_held() &&
7541                      !lockdep_commit_lock_is_held(net));
7542
7543         rcu_read_lock();
7544         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7545         if (!list)
7546                 goto out;
7547
7548         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7549                 if (objtype == obj->ops->type->type &&
7550                     nft_active_genmask(obj, genmask)) {
7551                         rcu_read_unlock();
7552                         return obj;
7553                 }
7554         }
7555 out:
7556         rcu_read_unlock();
7557         return ERR_PTR(-ENOENT);
7558 }
7559 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7560
7561 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7562                                                   const struct nlattr *nla,
7563                                                   u32 objtype, u8 genmask)
7564 {
7565         struct nft_object *obj;
7566
7567         list_for_each_entry(obj, &table->objects, list) {
7568                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7569                     objtype == obj->ops->type->type &&
7570                     nft_active_genmask(obj, genmask))
7571                         return obj;
7572         }
7573         return ERR_PTR(-ENOENT);
7574 }
7575
7576 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7577         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7578                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7579         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7580                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7581         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7582         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7583         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7584         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7585                                     .len = NFT_USERDATA_MAXLEN },
7586 };
7587
7588 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7589                                        const struct nft_object_type *type,
7590                                        const struct nlattr *attr)
7591 {
7592         struct nlattr **tb;
7593         const struct nft_object_ops *ops;
7594         struct nft_object *obj;
7595         int err = -ENOMEM;
7596
7597         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7598         if (!tb)
7599                 goto err1;
7600
7601         if (attr) {
7602                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7603                                                   type->policy, NULL);
7604                 if (err < 0)
7605                         goto err2;
7606         } else {
7607                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7608         }
7609
7610         if (type->select_ops) {
7611                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7612                 if (IS_ERR(ops)) {
7613                         err = PTR_ERR(ops);
7614                         goto err2;
7615                 }
7616         } else {
7617                 ops = type->ops;
7618         }
7619
7620         err = -ENOMEM;
7621         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7622         if (!obj)
7623                 goto err2;
7624
7625         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7626         if (err < 0)
7627                 goto err3;
7628
7629         obj->ops = ops;
7630
7631         kfree(tb);
7632         return obj;
7633 err3:
7634         kfree(obj);
7635 err2:
7636         kfree(tb);
7637 err1:
7638         return ERR_PTR(err);
7639 }
7640
7641 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7642                            struct nft_object *obj, bool reset)
7643 {
7644         struct nlattr *nest;
7645
7646         nest = nla_nest_start_noflag(skb, attr);
7647         if (!nest)
7648                 goto nla_put_failure;
7649         if (obj->ops->dump(skb, obj, reset) < 0)
7650                 goto nla_put_failure;
7651         nla_nest_end(skb, nest);
7652         return 0;
7653
7654 nla_put_failure:
7655         return -1;
7656 }
7657
7658 static const struct nft_object_type *__nft_obj_type_get(u32 objtype, u8 family)
7659 {
7660         const struct nft_object_type *type;
7661
7662         list_for_each_entry_rcu(type, &nf_tables_objects, list) {
7663                 if (type->family != NFPROTO_UNSPEC &&
7664                     type->family != family)
7665                         continue;
7666
7667                 if (objtype == type->type)
7668                         return type;
7669         }
7670         return NULL;
7671 }
7672
7673 static const struct nft_object_type *
7674 nft_obj_type_get(struct net *net, u32 objtype, u8 family)
7675 {
7676         const struct nft_object_type *type;
7677
7678         rcu_read_lock();
7679         type = __nft_obj_type_get(objtype, family);
7680         if (type != NULL && try_module_get(type->owner)) {
7681                 rcu_read_unlock();
7682                 return type;
7683         }
7684         rcu_read_unlock();
7685
7686         lockdep_nfnl_nft_mutex_not_held();
7687 #ifdef CONFIG_MODULES
7688         if (type == NULL) {
7689                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7690                         return ERR_PTR(-EAGAIN);
7691         }
7692 #endif
7693         return ERR_PTR(-ENOENT);
7694 }
7695
7696 static int nf_tables_updobj(const struct nft_ctx *ctx,
7697                             const struct nft_object_type *type,
7698                             const struct nlattr *attr,
7699                             struct nft_object *obj)
7700 {
7701         struct nft_object *newobj;
7702         struct nft_trans *trans;
7703         int err = -ENOMEM;
7704
7705         if (!try_module_get(type->owner))
7706                 return -ENOENT;
7707
7708         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7709                                 sizeof(struct nft_trans_obj));
7710         if (!trans)
7711                 goto err_trans;
7712
7713         newobj = nft_obj_init(ctx, type, attr);
7714         if (IS_ERR(newobj)) {
7715                 err = PTR_ERR(newobj);
7716                 goto err_free_trans;
7717         }
7718
7719         nft_trans_obj(trans) = obj;
7720         nft_trans_obj_update(trans) = true;
7721         nft_trans_obj_newobj(trans) = newobj;
7722         nft_trans_commit_list_add_tail(ctx->net, trans);
7723
7724         return 0;
7725
7726 err_free_trans:
7727         kfree(trans);
7728 err_trans:
7729         module_put(type->owner);
7730         return err;
7731 }
7732
7733 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7734                             const struct nlattr * const nla[])
7735 {
7736         struct netlink_ext_ack *extack = info->extack;
7737         u8 genmask = nft_genmask_next(info->net);
7738         u8 family = info->nfmsg->nfgen_family;
7739         const struct nft_object_type *type;
7740         struct net *net = info->net;
7741         struct nft_table *table;
7742         struct nft_object *obj;
7743         struct nft_ctx ctx;
7744         u32 objtype;
7745         int err;
7746
7747         if (!nla[NFTA_OBJ_TYPE] ||
7748             !nla[NFTA_OBJ_NAME] ||
7749             !nla[NFTA_OBJ_DATA])
7750                 return -EINVAL;
7751
7752         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7753                                  NETLINK_CB(skb).portid);
7754         if (IS_ERR(table)) {
7755                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7756                 return PTR_ERR(table);
7757         }
7758
7759         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7760         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7761         if (IS_ERR(obj)) {
7762                 err = PTR_ERR(obj);
7763                 if (err != -ENOENT) {
7764                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7765                         return err;
7766                 }
7767         } else {
7768                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7769                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7770                         return -EEXIST;
7771                 }
7772                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7773                         return -EOPNOTSUPP;
7774
7775                 type = __nft_obj_type_get(objtype, family);
7776                 if (WARN_ON_ONCE(!type))
7777                         return -ENOENT;
7778
7779                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7780
7781                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7782         }
7783
7784         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7785
7786         if (!nft_use_inc(&table->use))
7787                 return -EMFILE;
7788
7789         type = nft_obj_type_get(net, objtype, family);
7790         if (IS_ERR(type)) {
7791                 err = PTR_ERR(type);
7792                 goto err_type;
7793         }
7794
7795         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7796         if (IS_ERR(obj)) {
7797                 err = PTR_ERR(obj);
7798                 goto err_init;
7799         }
7800         obj->key.table = table;
7801         obj->handle = nf_tables_alloc_handle(table);
7802
7803         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7804         if (!obj->key.name) {
7805                 err = -ENOMEM;
7806                 goto err_strdup;
7807         }
7808
7809         if (nla[NFTA_OBJ_USERDATA]) {
7810                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7811                 if (obj->udata == NULL)
7812                         goto err_userdata;
7813
7814                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7815         }
7816
7817         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7818         if (err < 0)
7819                 goto err_trans;
7820
7821         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7822                               nft_objname_ht_params);
7823         if (err < 0)
7824                 goto err_obj_ht;
7825
7826         list_add_tail_rcu(&obj->list, &table->objects);
7827
7828         return 0;
7829 err_obj_ht:
7830         /* queued in transaction log */
7831         INIT_LIST_HEAD(&obj->list);
7832         return err;
7833 err_trans:
7834         kfree(obj->udata);
7835 err_userdata:
7836         kfree(obj->key.name);
7837 err_strdup:
7838         if (obj->ops->destroy)
7839                 obj->ops->destroy(&ctx, obj);
7840         kfree(obj);
7841 err_init:
7842         module_put(type->owner);
7843 err_type:
7844         nft_use_dec_restore(&table->use);
7845
7846         return err;
7847 }
7848
7849 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7850                                    u32 portid, u32 seq, int event, u32 flags,
7851                                    int family, const struct nft_table *table,
7852                                    struct nft_object *obj, bool reset)
7853 {
7854         struct nlmsghdr *nlh;
7855
7856         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7857         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7858                            NFNETLINK_V0, nft_base_seq(net));
7859         if (!nlh)
7860                 goto nla_put_failure;
7861
7862         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7863             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7864             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7865                          NFTA_OBJ_PAD))
7866                 goto nla_put_failure;
7867
7868         if (event == NFT_MSG_DELOBJ) {
7869                 nlmsg_end(skb, nlh);
7870                 return 0;
7871         }
7872
7873         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7874             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7875             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7876                 goto nla_put_failure;
7877
7878         if (obj->udata &&
7879             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7880                 goto nla_put_failure;
7881
7882         nlmsg_end(skb, nlh);
7883         return 0;
7884
7885 nla_put_failure:
7886         nlmsg_trim(skb, nlh);
7887         return -1;
7888 }
7889
7890 static void audit_log_obj_reset(const struct nft_table *table,
7891                                 unsigned int base_seq, unsigned int nentries)
7892 {
7893         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
7894
7895         audit_log_nfcfg(buf, table->family, nentries,
7896                         AUDIT_NFT_OP_OBJ_RESET, GFP_ATOMIC);
7897         kfree(buf);
7898 }
7899
7900 struct nft_obj_dump_ctx {
7901         unsigned int    s_idx;
7902         char            *table;
7903         u32             type;
7904         bool            reset;
7905 };
7906
7907 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7908 {
7909         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7910         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7911         struct net *net = sock_net(skb->sk);
7912         int family = nfmsg->nfgen_family;
7913         struct nftables_pernet *nft_net;
7914         const struct nft_table *table;
7915         unsigned int entries = 0;
7916         struct nft_object *obj;
7917         unsigned int idx = 0;
7918         int rc = 0;
7919
7920         rcu_read_lock();
7921         nft_net = nft_pernet(net);
7922         cb->seq = READ_ONCE(nft_net->base_seq);
7923
7924         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7925                 if (family != NFPROTO_UNSPEC && family != table->family)
7926                         continue;
7927
7928                 entries = 0;
7929                 list_for_each_entry_rcu(obj, &table->objects, list) {
7930                         if (!nft_is_active(net, obj))
7931                                 goto cont;
7932                         if (idx < ctx->s_idx)
7933                                 goto cont;
7934                         if (ctx->table && strcmp(ctx->table, table->name))
7935                                 goto cont;
7936                         if (ctx->type != NFT_OBJECT_UNSPEC &&
7937                             obj->ops->type->type != ctx->type)
7938                                 goto cont;
7939
7940                         rc = nf_tables_fill_obj_info(skb, net,
7941                                                      NETLINK_CB(cb->skb).portid,
7942                                                      cb->nlh->nlmsg_seq,
7943                                                      NFT_MSG_NEWOBJ,
7944                                                      NLM_F_MULTI | NLM_F_APPEND,
7945                                                      table->family, table,
7946                                                      obj, ctx->reset);
7947                         if (rc < 0)
7948                                 break;
7949
7950                         entries++;
7951                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7952 cont:
7953                         idx++;
7954                 }
7955                 if (ctx->reset && entries)
7956                         audit_log_obj_reset(table, nft_net->base_seq, entries);
7957                 if (rc < 0)
7958                         break;
7959         }
7960         rcu_read_unlock();
7961
7962         ctx->s_idx = idx;
7963         return skb->len;
7964 }
7965
7966 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7967 {
7968         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7969         const struct nlattr * const *nla = cb->data;
7970
7971         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
7972
7973         if (nla[NFTA_OBJ_TABLE]) {
7974                 ctx->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7975                 if (!ctx->table)
7976                         return -ENOMEM;
7977         }
7978
7979         if (nla[NFTA_OBJ_TYPE])
7980                 ctx->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7981
7982         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7983                 ctx->reset = true;
7984
7985         return 0;
7986 }
7987
7988 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7989 {
7990         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7991
7992         kfree(ctx->table);
7993
7994         return 0;
7995 }
7996
7997 /* called with rcu_read_lock held */
7998 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7999                             const struct nlattr * const nla[])
8000 {
8001         struct netlink_ext_ack *extack = info->extack;
8002         u8 genmask = nft_genmask_cur(info->net);
8003         u8 family = info->nfmsg->nfgen_family;
8004         const struct nft_table *table;
8005         struct net *net = info->net;
8006         struct nft_object *obj;
8007         struct sk_buff *skb2;
8008         bool reset = false;
8009         u32 objtype;
8010         int err;
8011
8012         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8013                 struct netlink_dump_control c = {
8014                         .start = nf_tables_dump_obj_start,
8015                         .dump = nf_tables_dump_obj,
8016                         .done = nf_tables_dump_obj_done,
8017                         .module = THIS_MODULE,
8018                         .data = (void *)nla,
8019                 };
8020
8021                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8022         }
8023
8024         if (!nla[NFTA_OBJ_NAME] ||
8025             !nla[NFTA_OBJ_TYPE])
8026                 return -EINVAL;
8027
8028         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
8029         if (IS_ERR(table)) {
8030                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
8031                 return PTR_ERR(table);
8032         }
8033
8034         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8035         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
8036         if (IS_ERR(obj)) {
8037                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
8038                 return PTR_ERR(obj);
8039         }
8040
8041         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8042         if (!skb2)
8043                 return -ENOMEM;
8044
8045         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
8046                 reset = true;
8047
8048         if (reset) {
8049                 const struct nftables_pernet *nft_net;
8050                 char *buf;
8051
8052                 nft_net = nft_pernet(net);
8053                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
8054
8055                 audit_log_nfcfg(buf,
8056                                 family,
8057                                 1,
8058                                 AUDIT_NFT_OP_OBJ_RESET,
8059                                 GFP_ATOMIC);
8060                 kfree(buf);
8061         }
8062
8063         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
8064                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
8065                                       family, table, obj, reset);
8066         if (err < 0)
8067                 goto err_fill_obj_info;
8068
8069         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8070
8071 err_fill_obj_info:
8072         kfree_skb(skb2);
8073         return err;
8074 }
8075
8076 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
8077 {
8078         if (obj->ops->destroy)
8079                 obj->ops->destroy(ctx, obj);
8080
8081         module_put(obj->ops->type->owner);
8082         kfree(obj->key.name);
8083         kfree(obj->udata);
8084         kfree(obj);
8085 }
8086
8087 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
8088                             const struct nlattr * const nla[])
8089 {
8090         struct netlink_ext_ack *extack = info->extack;
8091         u8 genmask = nft_genmask_next(info->net);
8092         u8 family = info->nfmsg->nfgen_family;
8093         struct net *net = info->net;
8094         const struct nlattr *attr;
8095         struct nft_table *table;
8096         struct nft_object *obj;
8097         struct nft_ctx ctx;
8098         u32 objtype;
8099
8100         if (!nla[NFTA_OBJ_TYPE] ||
8101             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
8102                 return -EINVAL;
8103
8104         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
8105                                  NETLINK_CB(skb).portid);
8106         if (IS_ERR(table)) {
8107                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
8108                 return PTR_ERR(table);
8109         }
8110
8111         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8112         if (nla[NFTA_OBJ_HANDLE]) {
8113                 attr = nla[NFTA_OBJ_HANDLE];
8114                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
8115         } else {
8116                 attr = nla[NFTA_OBJ_NAME];
8117                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
8118         }
8119
8120         if (IS_ERR(obj)) {
8121                 if (PTR_ERR(obj) == -ENOENT &&
8122                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
8123                         return 0;
8124
8125                 NL_SET_BAD_ATTR(extack, attr);
8126                 return PTR_ERR(obj);
8127         }
8128         if (obj->use > 0) {
8129                 NL_SET_BAD_ATTR(extack, attr);
8130                 return -EBUSY;
8131         }
8132
8133         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8134
8135         return nft_delobj(&ctx, obj);
8136 }
8137
8138 static void
8139 __nft_obj_notify(struct net *net, const struct nft_table *table,
8140                  struct nft_object *obj, u32 portid, u32 seq, int event,
8141                  u16 flags, int family, int report, gfp_t gfp)
8142 {
8143         struct nftables_pernet *nft_net = nft_pernet(net);
8144         struct sk_buff *skb;
8145         int err;
8146
8147         if (!report &&
8148             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8149                 return;
8150
8151         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
8152         if (skb == NULL)
8153                 goto err;
8154
8155         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
8156                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
8157                                       family, table, obj, false);
8158         if (err < 0) {
8159                 kfree_skb(skb);
8160                 goto err;
8161         }
8162
8163         nft_notify_enqueue(skb, report, &nft_net->notify_list);
8164         return;
8165 err:
8166         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
8167 }
8168
8169 void nft_obj_notify(struct net *net, const struct nft_table *table,
8170                     struct nft_object *obj, u32 portid, u32 seq, int event,
8171                     u16 flags, int family, int report, gfp_t gfp)
8172 {
8173         struct nftables_pernet *nft_net = nft_pernet(net);
8174         char *buf = kasprintf(gfp, "%s:%u",
8175                               table->name, nft_net->base_seq);
8176
8177         audit_log_nfcfg(buf,
8178                         family,
8179                         obj->handle,
8180                         event == NFT_MSG_NEWOBJ ?
8181                                  AUDIT_NFT_OP_OBJ_REGISTER :
8182                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
8183                         gfp);
8184         kfree(buf);
8185
8186         __nft_obj_notify(net, table, obj, portid, seq, event,
8187                          flags, family, report, gfp);
8188 }
8189 EXPORT_SYMBOL_GPL(nft_obj_notify);
8190
8191 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
8192                                  struct nft_object *obj, int event)
8193 {
8194         __nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid,
8195                          ctx->seq, event, ctx->flags, ctx->family,
8196                          ctx->report, GFP_KERNEL);
8197 }
8198
8199 /*
8200  * Flow tables
8201  */
8202 void nft_register_flowtable_type(struct nf_flowtable_type *type)
8203 {
8204         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8205         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
8206         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8207 }
8208 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
8209
8210 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
8211 {
8212         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8213         list_del_rcu(&type->list);
8214         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8215 }
8216 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
8217
8218 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
8219         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
8220                                             .len = NFT_NAME_MAXLEN - 1 },
8221         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
8222                                             .len = NFT_NAME_MAXLEN - 1 },
8223         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
8224         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
8225         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
8226 };
8227
8228 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
8229                                            const struct nlattr *nla, u8 genmask)
8230 {
8231         struct nft_flowtable *flowtable;
8232
8233         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8234                 if (!nla_strcmp(nla, flowtable->name) &&
8235                     nft_active_genmask(flowtable, genmask))
8236                         return flowtable;
8237         }
8238         return ERR_PTR(-ENOENT);
8239 }
8240 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
8241
8242 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
8243                                     struct nft_flowtable *flowtable,
8244                                     enum nft_trans_phase phase)
8245 {
8246         switch (phase) {
8247         case NFT_TRANS_PREPARE_ERROR:
8248         case NFT_TRANS_PREPARE:
8249         case NFT_TRANS_ABORT:
8250         case NFT_TRANS_RELEASE:
8251                 nft_use_dec(&flowtable->use);
8252                 fallthrough;
8253         default:
8254                 return;
8255         }
8256 }
8257 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
8258
8259 static struct nft_flowtable *
8260 nft_flowtable_lookup_byhandle(const struct nft_table *table,
8261                               const struct nlattr *nla, u8 genmask)
8262 {
8263        struct nft_flowtable *flowtable;
8264
8265        list_for_each_entry(flowtable, &table->flowtables, list) {
8266                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
8267                    nft_active_genmask(flowtable, genmask))
8268                        return flowtable;
8269        }
8270        return ERR_PTR(-ENOENT);
8271 }
8272
8273 struct nft_flowtable_hook {
8274         u32                     num;
8275         int                     priority;
8276         struct list_head        list;
8277 };
8278
8279 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
8280         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
8281         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
8282         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
8283 };
8284
8285 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
8286                                     const struct nlattr * const nla[],
8287                                     struct nft_flowtable_hook *flowtable_hook,
8288                                     struct nft_flowtable *flowtable,
8289                                     struct netlink_ext_ack *extack, bool add)
8290 {
8291         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
8292         struct nft_hook *hook;
8293         int hooknum, priority;
8294         int err;
8295
8296         INIT_LIST_HEAD(&flowtable_hook->list);
8297
8298         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
8299                                           nla[NFTA_FLOWTABLE_HOOK],
8300                                           nft_flowtable_hook_policy, NULL);
8301         if (err < 0)
8302                 return err;
8303
8304         if (add) {
8305                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
8306                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8307                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8308                         return -ENOENT;
8309                 }
8310
8311                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8312                 if (hooknum != NF_NETDEV_INGRESS)
8313                         return -EOPNOTSUPP;
8314
8315                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8316
8317                 flowtable_hook->priority        = priority;
8318                 flowtable_hook->num             = hooknum;
8319         } else {
8320                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
8321                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8322                         if (hooknum != flowtable->hooknum)
8323                                 return -EOPNOTSUPP;
8324                 }
8325
8326                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8327                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8328                         if (priority != flowtable->data.priority)
8329                                 return -EOPNOTSUPP;
8330                 }
8331
8332                 flowtable_hook->priority        = flowtable->data.priority;
8333                 flowtable_hook->num             = flowtable->hooknum;
8334         }
8335
8336         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
8337                 err = nf_tables_parse_netdev_hooks(ctx->net,
8338                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
8339                                                    &flowtable_hook->list,
8340                                                    extack);
8341                 if (err < 0)
8342                         return err;
8343         }
8344
8345         list_for_each_entry(hook, &flowtable_hook->list, list) {
8346                 hook->ops.pf            = NFPROTO_NETDEV;
8347                 hook->ops.hooknum       = flowtable_hook->num;
8348                 hook->ops.priority      = flowtable_hook->priority;
8349                 hook->ops.priv          = &flowtable->data;
8350                 hook->ops.hook          = flowtable->data.type->hook;
8351         }
8352
8353         return err;
8354 }
8355
8356 /* call under rcu_read_lock */
8357 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
8358 {
8359         const struct nf_flowtable_type *type;
8360
8361         list_for_each_entry_rcu(type, &nf_tables_flowtables, list) {
8362                 if (family == type->family)
8363                         return type;
8364         }
8365         return NULL;
8366 }
8367
8368 static const struct nf_flowtable_type *
8369 nft_flowtable_type_get(struct net *net, u8 family)
8370 {
8371         const struct nf_flowtable_type *type;
8372
8373         rcu_read_lock();
8374         type = __nft_flowtable_type_get(family);
8375         if (type != NULL && try_module_get(type->owner)) {
8376                 rcu_read_unlock();
8377                 return type;
8378         }
8379         rcu_read_unlock();
8380
8381         lockdep_nfnl_nft_mutex_not_held();
8382 #ifdef CONFIG_MODULES
8383         if (type == NULL) {
8384                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
8385                         return ERR_PTR(-EAGAIN);
8386         }
8387 #endif
8388         return ERR_PTR(-ENOENT);
8389 }
8390
8391 /* Only called from error and netdev event paths. */
8392 static void nft_unregister_flowtable_hook(struct net *net,
8393                                           struct nft_flowtable *flowtable,
8394                                           struct nft_hook *hook)
8395 {
8396         nf_unregister_net_hook(net, &hook->ops);
8397         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8398                                     FLOW_BLOCK_UNBIND);
8399 }
8400
8401 static void __nft_unregister_flowtable_net_hooks(struct net *net,
8402                                                  struct list_head *hook_list,
8403                                                  bool release_netdev)
8404 {
8405         struct nft_hook *hook, *next;
8406
8407         list_for_each_entry_safe(hook, next, hook_list, list) {
8408                 nf_unregister_net_hook(net, &hook->ops);
8409                 if (release_netdev) {
8410                         list_del(&hook->list);
8411                         kfree_rcu(hook, rcu);
8412                 }
8413         }
8414 }
8415
8416 static void nft_unregister_flowtable_net_hooks(struct net *net,
8417                                                struct list_head *hook_list)
8418 {
8419         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
8420 }
8421
8422 static int nft_register_flowtable_net_hooks(struct net *net,
8423                                             struct nft_table *table,
8424                                             struct list_head *hook_list,
8425                                             struct nft_flowtable *flowtable)
8426 {
8427         struct nft_hook *hook, *hook2, *next;
8428         struct nft_flowtable *ft;
8429         int err, i = 0;
8430
8431         list_for_each_entry(hook, hook_list, list) {
8432                 list_for_each_entry(ft, &table->flowtables, list) {
8433                         if (!nft_is_active_next(net, ft))
8434                                 continue;
8435
8436                         list_for_each_entry(hook2, &ft->hook_list, list) {
8437                                 if (hook->ops.dev == hook2->ops.dev &&
8438                                     hook->ops.pf == hook2->ops.pf) {
8439                                         err = -EEXIST;
8440                                         goto err_unregister_net_hooks;
8441                                 }
8442                         }
8443                 }
8444
8445                 err = flowtable->data.type->setup(&flowtable->data,
8446                                                   hook->ops.dev,
8447                                                   FLOW_BLOCK_BIND);
8448                 if (err < 0)
8449                         goto err_unregister_net_hooks;
8450
8451                 err = nf_register_net_hook(net, &hook->ops);
8452                 if (err < 0) {
8453                         flowtable->data.type->setup(&flowtable->data,
8454                                                     hook->ops.dev,
8455                                                     FLOW_BLOCK_UNBIND);
8456                         goto err_unregister_net_hooks;
8457                 }
8458
8459                 i++;
8460         }
8461
8462         return 0;
8463
8464 err_unregister_net_hooks:
8465         list_for_each_entry_safe(hook, next, hook_list, list) {
8466                 if (i-- <= 0)
8467                         break;
8468
8469                 nft_unregister_flowtable_hook(net, flowtable, hook);
8470                 list_del_rcu(&hook->list);
8471                 kfree_rcu(hook, rcu);
8472         }
8473
8474         return err;
8475 }
8476
8477 static void nft_hooks_destroy(struct list_head *hook_list)
8478 {
8479         struct nft_hook *hook, *next;
8480
8481         list_for_each_entry_safe(hook, next, hook_list, list) {
8482                 list_del_rcu(&hook->list);
8483                 kfree_rcu(hook, rcu);
8484         }
8485 }
8486
8487 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
8488                                 struct nft_flowtable *flowtable,
8489                                 struct netlink_ext_ack *extack)
8490 {
8491         const struct nlattr * const *nla = ctx->nla;
8492         struct nft_flowtable_hook flowtable_hook;
8493         struct nft_hook *hook, *next;
8494         struct nft_trans *trans;
8495         bool unregister = false;
8496         u32 flags;
8497         int err;
8498
8499         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8500                                        extack, false);
8501         if (err < 0)
8502                 return err;
8503
8504         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8505                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
8506                         list_del(&hook->list);
8507                         kfree(hook);
8508                 }
8509         }
8510
8511         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8512                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8513                 if (flags & ~NFT_FLOWTABLE_MASK) {
8514                         err = -EOPNOTSUPP;
8515                         goto err_flowtable_update_hook;
8516                 }
8517                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8518                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8519                         err = -EOPNOTSUPP;
8520                         goto err_flowtable_update_hook;
8521                 }
8522         } else {
8523                 flags = flowtable->data.flags;
8524         }
8525
8526         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8527                                                &flowtable_hook.list, flowtable);
8528         if (err < 0)
8529                 goto err_flowtable_update_hook;
8530
8531         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8532                                 sizeof(struct nft_trans_flowtable));
8533         if (!trans) {
8534                 unregister = true;
8535                 err = -ENOMEM;
8536                 goto err_flowtable_update_hook;
8537         }
8538
8539         nft_trans_flowtable_flags(trans) = flags;
8540         nft_trans_flowtable(trans) = flowtable;
8541         nft_trans_flowtable_update(trans) = true;
8542         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8543         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8544
8545         nft_trans_commit_list_add_tail(ctx->net, trans);
8546
8547         return 0;
8548
8549 err_flowtable_update_hook:
8550         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8551                 if (unregister)
8552                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8553                 list_del_rcu(&hook->list);
8554                 kfree_rcu(hook, rcu);
8555         }
8556
8557         return err;
8558
8559 }
8560
8561 static int nf_tables_newflowtable(struct sk_buff *skb,
8562                                   const struct nfnl_info *info,
8563                                   const struct nlattr * const nla[])
8564 {
8565         struct netlink_ext_ack *extack = info->extack;
8566         struct nft_flowtable_hook flowtable_hook;
8567         u8 genmask = nft_genmask_next(info->net);
8568         u8 family = info->nfmsg->nfgen_family;
8569         const struct nf_flowtable_type *type;
8570         struct nft_flowtable *flowtable;
8571         struct net *net = info->net;
8572         struct nft_table *table;
8573         struct nft_trans *trans;
8574         struct nft_ctx ctx;
8575         int err;
8576
8577         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8578             !nla[NFTA_FLOWTABLE_NAME] ||
8579             !nla[NFTA_FLOWTABLE_HOOK])
8580                 return -EINVAL;
8581
8582         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8583                                  genmask, NETLINK_CB(skb).portid);
8584         if (IS_ERR(table)) {
8585                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8586                 return PTR_ERR(table);
8587         }
8588
8589         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8590                                          genmask);
8591         if (IS_ERR(flowtable)) {
8592                 err = PTR_ERR(flowtable);
8593                 if (err != -ENOENT) {
8594                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8595                         return err;
8596                 }
8597         } else {
8598                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8599                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8600                         return -EEXIST;
8601                 }
8602
8603                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8604
8605                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8606         }
8607
8608         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8609
8610         if (!nft_use_inc(&table->use))
8611                 return -EMFILE;
8612
8613         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8614         if (!flowtable) {
8615                 err = -ENOMEM;
8616                 goto flowtable_alloc;
8617         }
8618
8619         flowtable->table = table;
8620         flowtable->handle = nf_tables_alloc_handle(table);
8621         INIT_LIST_HEAD(&flowtable->hook_list);
8622
8623         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8624         if (!flowtable->name) {
8625                 err = -ENOMEM;
8626                 goto err1;
8627         }
8628
8629         type = nft_flowtable_type_get(net, family);
8630         if (IS_ERR(type)) {
8631                 err = PTR_ERR(type);
8632                 goto err2;
8633         }
8634
8635         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8636                 flowtable->data.flags =
8637                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8638                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8639                         err = -EOPNOTSUPP;
8640                         goto err3;
8641                 }
8642         }
8643
8644         write_pnet(&flowtable->data.net, net);
8645         flowtable->data.type = type;
8646         err = type->init(&flowtable->data);
8647         if (err < 0)
8648                 goto err3;
8649
8650         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8651                                        extack, true);
8652         if (err < 0)
8653                 goto err_flowtable_parse_hooks;
8654
8655         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8656         flowtable->data.priority = flowtable_hook.priority;
8657         flowtable->hooknum = flowtable_hook.num;
8658
8659         trans = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8660         if (IS_ERR(trans)) {
8661                 err = PTR_ERR(trans);
8662                 goto err_flowtable_trans;
8663         }
8664
8665         /* This must be LAST to ensure no packets are walking over this flowtable. */
8666         err = nft_register_flowtable_net_hooks(ctx.net, table,
8667                                                &flowtable->hook_list,
8668                                                flowtable);
8669         if (err < 0)
8670                 goto err_flowtable_hooks;
8671
8672         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8673
8674         return 0;
8675
8676 err_flowtable_hooks:
8677         nft_trans_destroy(trans);
8678 err_flowtable_trans:
8679         nft_hooks_destroy(&flowtable->hook_list);
8680 err_flowtable_parse_hooks:
8681         flowtable->data.type->free(&flowtable->data);
8682 err3:
8683         module_put(type->owner);
8684 err2:
8685         kfree(flowtable->name);
8686 err1:
8687         kfree(flowtable);
8688 flowtable_alloc:
8689         nft_use_dec_restore(&table->use);
8690
8691         return err;
8692 }
8693
8694 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8695 {
8696         struct nft_hook *this, *next;
8697
8698         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8699                 list_del(&this->list);
8700                 kfree(this);
8701         }
8702 }
8703
8704 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8705                                  struct nft_flowtable *flowtable,
8706                                  struct netlink_ext_ack *extack)
8707 {
8708         const struct nlattr * const *nla = ctx->nla;
8709         struct nft_flowtable_hook flowtable_hook;
8710         LIST_HEAD(flowtable_del_list);
8711         struct nft_hook *this, *hook;
8712         struct nft_trans *trans;
8713         int err;
8714
8715         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8716                                        extack, false);
8717         if (err < 0)
8718                 return err;
8719
8720         list_for_each_entry(this, &flowtable_hook.list, list) {
8721                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8722                 if (!hook) {
8723                         err = -ENOENT;
8724                         goto err_flowtable_del_hook;
8725                 }
8726                 list_move(&hook->list, &flowtable_del_list);
8727         }
8728
8729         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8730                                 sizeof(struct nft_trans_flowtable));
8731         if (!trans) {
8732                 err = -ENOMEM;
8733                 goto err_flowtable_del_hook;
8734         }
8735
8736         nft_trans_flowtable(trans) = flowtable;
8737         nft_trans_flowtable_update(trans) = true;
8738         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8739         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8740         nft_flowtable_hook_release(&flowtable_hook);
8741
8742         nft_trans_commit_list_add_tail(ctx->net, trans);
8743
8744         return 0;
8745
8746 err_flowtable_del_hook:
8747         list_splice(&flowtable_del_list, &flowtable->hook_list);
8748         nft_flowtable_hook_release(&flowtable_hook);
8749
8750         return err;
8751 }
8752
8753 static int nf_tables_delflowtable(struct sk_buff *skb,
8754                                   const struct nfnl_info *info,
8755                                   const struct nlattr * const nla[])
8756 {
8757         struct netlink_ext_ack *extack = info->extack;
8758         u8 genmask = nft_genmask_next(info->net);
8759         u8 family = info->nfmsg->nfgen_family;
8760         struct nft_flowtable *flowtable;
8761         struct net *net = info->net;
8762         const struct nlattr *attr;
8763         struct nft_table *table;
8764         struct nft_ctx ctx;
8765
8766         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8767             (!nla[NFTA_FLOWTABLE_NAME] &&
8768              !nla[NFTA_FLOWTABLE_HANDLE]))
8769                 return -EINVAL;
8770
8771         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8772                                  genmask, NETLINK_CB(skb).portid);
8773         if (IS_ERR(table)) {
8774                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8775                 return PTR_ERR(table);
8776         }
8777
8778         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8779                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8780                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8781         } else {
8782                 attr = nla[NFTA_FLOWTABLE_NAME];
8783                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8784         }
8785
8786         if (IS_ERR(flowtable)) {
8787                 if (PTR_ERR(flowtable) == -ENOENT &&
8788                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8789                         return 0;
8790
8791                 NL_SET_BAD_ATTR(extack, attr);
8792                 return PTR_ERR(flowtable);
8793         }
8794
8795         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8796
8797         if (nla[NFTA_FLOWTABLE_HOOK])
8798                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8799
8800         if (flowtable->use > 0) {
8801                 NL_SET_BAD_ATTR(extack, attr);
8802                 return -EBUSY;
8803         }
8804
8805         return nft_delflowtable(&ctx, flowtable);
8806 }
8807
8808 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8809                                          u32 portid, u32 seq, int event,
8810                                          u32 flags, int family,
8811                                          struct nft_flowtable *flowtable,
8812                                          struct list_head *hook_list)
8813 {
8814         struct nlattr *nest, *nest_devs;
8815         struct nft_hook *hook;
8816         struct nlmsghdr *nlh;
8817
8818         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8819         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8820                            NFNETLINK_V0, nft_base_seq(net));
8821         if (!nlh)
8822                 goto nla_put_failure;
8823
8824         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8825             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8826             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8827                          NFTA_FLOWTABLE_PAD))
8828                 goto nla_put_failure;
8829
8830         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8831                 nlmsg_end(skb, nlh);
8832                 return 0;
8833         }
8834
8835         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8836             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8837                 goto nla_put_failure;
8838
8839         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8840         if (!nest)
8841                 goto nla_put_failure;
8842         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8843             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8844                 goto nla_put_failure;
8845
8846         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8847         if (!nest_devs)
8848                 goto nla_put_failure;
8849
8850         if (!hook_list)
8851                 hook_list = &flowtable->hook_list;
8852
8853         list_for_each_entry_rcu(hook, hook_list, list) {
8854                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8855                         goto nla_put_failure;
8856         }
8857         nla_nest_end(skb, nest_devs);
8858         nla_nest_end(skb, nest);
8859
8860         nlmsg_end(skb, nlh);
8861         return 0;
8862
8863 nla_put_failure:
8864         nlmsg_trim(skb, nlh);
8865         return -1;
8866 }
8867
8868 struct nft_flowtable_filter {
8869         char            *table;
8870 };
8871
8872 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8873                                     struct netlink_callback *cb)
8874 {
8875         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8876         struct nft_flowtable_filter *filter = cb->data;
8877         unsigned int idx = 0, s_idx = cb->args[0];
8878         struct net *net = sock_net(skb->sk);
8879         int family = nfmsg->nfgen_family;
8880         struct nft_flowtable *flowtable;
8881         struct nftables_pernet *nft_net;
8882         const struct nft_table *table;
8883
8884         rcu_read_lock();
8885         nft_net = nft_pernet(net);
8886         cb->seq = READ_ONCE(nft_net->base_seq);
8887
8888         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8889                 if (family != NFPROTO_UNSPEC && family != table->family)
8890                         continue;
8891
8892                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8893                         if (!nft_is_active(net, flowtable))
8894                                 goto cont;
8895                         if (idx < s_idx)
8896                                 goto cont;
8897                         if (idx > s_idx)
8898                                 memset(&cb->args[1], 0,
8899                                        sizeof(cb->args) - sizeof(cb->args[0]));
8900                         if (filter && filter->table &&
8901                             strcmp(filter->table, table->name))
8902                                 goto cont;
8903
8904                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8905                                                           cb->nlh->nlmsg_seq,
8906                                                           NFT_MSG_NEWFLOWTABLE,
8907                                                           NLM_F_MULTI | NLM_F_APPEND,
8908                                                           table->family,
8909                                                           flowtable, NULL) < 0)
8910                                 goto done;
8911
8912                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8913 cont:
8914                         idx++;
8915                 }
8916         }
8917 done:
8918         rcu_read_unlock();
8919
8920         cb->args[0] = idx;
8921         return skb->len;
8922 }
8923
8924 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8925 {
8926         const struct nlattr * const *nla = cb->data;
8927         struct nft_flowtable_filter *filter = NULL;
8928
8929         if (nla[NFTA_FLOWTABLE_TABLE]) {
8930                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8931                 if (!filter)
8932                         return -ENOMEM;
8933
8934                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8935                                            GFP_ATOMIC);
8936                 if (!filter->table) {
8937                         kfree(filter);
8938                         return -ENOMEM;
8939                 }
8940         }
8941
8942         cb->data = filter;
8943         return 0;
8944 }
8945
8946 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8947 {
8948         struct nft_flowtable_filter *filter = cb->data;
8949
8950         if (!filter)
8951                 return 0;
8952
8953         kfree(filter->table);
8954         kfree(filter);
8955
8956         return 0;
8957 }
8958
8959 /* called with rcu_read_lock held */
8960 static int nf_tables_getflowtable(struct sk_buff *skb,
8961                                   const struct nfnl_info *info,
8962                                   const struct nlattr * const nla[])
8963 {
8964         struct netlink_ext_ack *extack = info->extack;
8965         u8 genmask = nft_genmask_cur(info->net);
8966         u8 family = info->nfmsg->nfgen_family;
8967         struct nft_flowtable *flowtable;
8968         const struct nft_table *table;
8969         struct net *net = info->net;
8970         struct sk_buff *skb2;
8971         int err;
8972
8973         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8974                 struct netlink_dump_control c = {
8975                         .start = nf_tables_dump_flowtable_start,
8976                         .dump = nf_tables_dump_flowtable,
8977                         .done = nf_tables_dump_flowtable_done,
8978                         .module = THIS_MODULE,
8979                         .data = (void *)nla,
8980                 };
8981
8982                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8983         }
8984
8985         if (!nla[NFTA_FLOWTABLE_NAME])
8986                 return -EINVAL;
8987
8988         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8989                                  genmask, 0);
8990         if (IS_ERR(table)) {
8991                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8992                 return PTR_ERR(table);
8993         }
8994
8995         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8996                                          genmask);
8997         if (IS_ERR(flowtable)) {
8998                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8999                 return PTR_ERR(flowtable);
9000         }
9001
9002         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
9003         if (!skb2)
9004                 return -ENOMEM;
9005
9006         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
9007                                             info->nlh->nlmsg_seq,
9008                                             NFT_MSG_NEWFLOWTABLE, 0, family,
9009                                             flowtable, NULL);
9010         if (err < 0)
9011                 goto err_fill_flowtable_info;
9012
9013         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
9014
9015 err_fill_flowtable_info:
9016         kfree_skb(skb2);
9017         return err;
9018 }
9019
9020 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
9021                                        struct nft_flowtable *flowtable,
9022                                        struct list_head *hook_list, int event)
9023 {
9024         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
9025         struct sk_buff *skb;
9026         u16 flags = 0;
9027         int err;
9028
9029         if (!ctx->report &&
9030             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
9031                 return;
9032
9033         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
9034         if (skb == NULL)
9035                 goto err;
9036
9037         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
9038                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
9039
9040         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
9041                                             ctx->seq, event, flags,
9042                                             ctx->family, flowtable, hook_list);
9043         if (err < 0) {
9044                 kfree_skb(skb);
9045                 goto err;
9046         }
9047
9048         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
9049         return;
9050 err:
9051         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
9052 }
9053
9054 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
9055 {
9056         struct nft_hook *hook, *next;
9057
9058         flowtable->data.type->free(&flowtable->data);
9059         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
9060                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
9061                                             FLOW_BLOCK_UNBIND);
9062                 list_del_rcu(&hook->list);
9063                 kfree(hook);
9064         }
9065         kfree(flowtable->name);
9066         module_put(flowtable->data.type->owner);
9067         kfree(flowtable);
9068 }
9069
9070 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
9071                                    u32 portid, u32 seq)
9072 {
9073         struct nftables_pernet *nft_net = nft_pernet(net);
9074         struct nlmsghdr *nlh;
9075         char buf[TASK_COMM_LEN];
9076         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
9077
9078         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
9079                            NFNETLINK_V0, nft_base_seq(net));
9080         if (!nlh)
9081                 goto nla_put_failure;
9082
9083         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
9084             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
9085             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
9086                 goto nla_put_failure;
9087
9088         nlmsg_end(skb, nlh);
9089         return 0;
9090
9091 nla_put_failure:
9092         nlmsg_trim(skb, nlh);
9093         return -EMSGSIZE;
9094 }
9095
9096 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
9097                                 struct nft_flowtable *flowtable)
9098 {
9099         struct nft_hook *hook;
9100
9101         list_for_each_entry(hook, &flowtable->hook_list, list) {
9102                 if (hook->ops.dev != dev)
9103                         continue;
9104
9105                 /* flow_offload_netdev_event() cleans up entries for us. */
9106                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
9107                 list_del_rcu(&hook->list);
9108                 kfree_rcu(hook, rcu);
9109                 break;
9110         }
9111 }
9112
9113 static int nf_tables_flowtable_event(struct notifier_block *this,
9114                                      unsigned long event, void *ptr)
9115 {
9116         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
9117         struct nft_flowtable *flowtable;
9118         struct nftables_pernet *nft_net;
9119         struct nft_table *table;
9120         struct net *net;
9121
9122         if (event != NETDEV_UNREGISTER)
9123                 return 0;
9124
9125         net = dev_net(dev);
9126         nft_net = nft_pernet(net);
9127         mutex_lock(&nft_net->commit_mutex);
9128         list_for_each_entry(table, &nft_net->tables, list) {
9129                 list_for_each_entry(flowtable, &table->flowtables, list) {
9130                         nft_flowtable_event(event, dev, flowtable);
9131                 }
9132         }
9133         mutex_unlock(&nft_net->commit_mutex);
9134
9135         return NOTIFY_DONE;
9136 }
9137
9138 static struct notifier_block nf_tables_flowtable_notifier = {
9139         .notifier_call  = nf_tables_flowtable_event,
9140 };
9141
9142 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
9143                                  int event)
9144 {
9145         struct nlmsghdr *nlh = nlmsg_hdr(skb);
9146         struct sk_buff *skb2;
9147         int err;
9148
9149         if (!nlmsg_report(nlh) &&
9150             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
9151                 return;
9152
9153         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
9154         if (skb2 == NULL)
9155                 goto err;
9156
9157         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
9158                                       nlh->nlmsg_seq);
9159         if (err < 0) {
9160                 kfree_skb(skb2);
9161                 goto err;
9162         }
9163
9164         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9165                        nlmsg_report(nlh), GFP_KERNEL);
9166         return;
9167 err:
9168         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9169                           -ENOBUFS);
9170 }
9171
9172 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
9173                             const struct nlattr * const nla[])
9174 {
9175         struct sk_buff *skb2;
9176         int err;
9177
9178         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
9179         if (skb2 == NULL)
9180                 return -ENOMEM;
9181
9182         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
9183                                       info->nlh->nlmsg_seq);
9184         if (err < 0)
9185                 goto err_fill_gen_info;
9186
9187         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
9188
9189 err_fill_gen_info:
9190         kfree_skb(skb2);
9191         return err;
9192 }
9193
9194 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
9195         [NFT_MSG_NEWTABLE] = {
9196                 .call           = nf_tables_newtable,
9197                 .type           = NFNL_CB_BATCH,
9198                 .attr_count     = NFTA_TABLE_MAX,
9199                 .policy         = nft_table_policy,
9200         },
9201         [NFT_MSG_GETTABLE] = {
9202                 .call           = nf_tables_gettable,
9203                 .type           = NFNL_CB_RCU,
9204                 .attr_count     = NFTA_TABLE_MAX,
9205                 .policy         = nft_table_policy,
9206         },
9207         [NFT_MSG_DELTABLE] = {
9208                 .call           = nf_tables_deltable,
9209                 .type           = NFNL_CB_BATCH,
9210                 .attr_count     = NFTA_TABLE_MAX,
9211                 .policy         = nft_table_policy,
9212         },
9213         [NFT_MSG_DESTROYTABLE] = {
9214                 .call           = nf_tables_deltable,
9215                 .type           = NFNL_CB_BATCH,
9216                 .attr_count     = NFTA_TABLE_MAX,
9217                 .policy         = nft_table_policy,
9218         },
9219         [NFT_MSG_NEWCHAIN] = {
9220                 .call           = nf_tables_newchain,
9221                 .type           = NFNL_CB_BATCH,
9222                 .attr_count     = NFTA_CHAIN_MAX,
9223                 .policy         = nft_chain_policy,
9224         },
9225         [NFT_MSG_GETCHAIN] = {
9226                 .call           = nf_tables_getchain,
9227                 .type           = NFNL_CB_RCU,
9228                 .attr_count     = NFTA_CHAIN_MAX,
9229                 .policy         = nft_chain_policy,
9230         },
9231         [NFT_MSG_DELCHAIN] = {
9232                 .call           = nf_tables_delchain,
9233                 .type           = NFNL_CB_BATCH,
9234                 .attr_count     = NFTA_CHAIN_MAX,
9235                 .policy         = nft_chain_policy,
9236         },
9237         [NFT_MSG_DESTROYCHAIN] = {
9238                 .call           = nf_tables_delchain,
9239                 .type           = NFNL_CB_BATCH,
9240                 .attr_count     = NFTA_CHAIN_MAX,
9241                 .policy         = nft_chain_policy,
9242         },
9243         [NFT_MSG_NEWRULE] = {
9244                 .call           = nf_tables_newrule,
9245                 .type           = NFNL_CB_BATCH,
9246                 .attr_count     = NFTA_RULE_MAX,
9247                 .policy         = nft_rule_policy,
9248         },
9249         [NFT_MSG_GETRULE] = {
9250                 .call           = nf_tables_getrule,
9251                 .type           = NFNL_CB_RCU,
9252                 .attr_count     = NFTA_RULE_MAX,
9253                 .policy         = nft_rule_policy,
9254         },
9255         [NFT_MSG_GETRULE_RESET] = {
9256                 .call           = nf_tables_getrule_reset,
9257                 .type           = NFNL_CB_RCU,
9258                 .attr_count     = NFTA_RULE_MAX,
9259                 .policy         = nft_rule_policy,
9260         },
9261         [NFT_MSG_DELRULE] = {
9262                 .call           = nf_tables_delrule,
9263                 .type           = NFNL_CB_BATCH,
9264                 .attr_count     = NFTA_RULE_MAX,
9265                 .policy         = nft_rule_policy,
9266         },
9267         [NFT_MSG_DESTROYRULE] = {
9268                 .call           = nf_tables_delrule,
9269                 .type           = NFNL_CB_BATCH,
9270                 .attr_count     = NFTA_RULE_MAX,
9271                 .policy         = nft_rule_policy,
9272         },
9273         [NFT_MSG_NEWSET] = {
9274                 .call           = nf_tables_newset,
9275                 .type           = NFNL_CB_BATCH,
9276                 .attr_count     = NFTA_SET_MAX,
9277                 .policy         = nft_set_policy,
9278         },
9279         [NFT_MSG_GETSET] = {
9280                 .call           = nf_tables_getset,
9281                 .type           = NFNL_CB_RCU,
9282                 .attr_count     = NFTA_SET_MAX,
9283                 .policy         = nft_set_policy,
9284         },
9285         [NFT_MSG_DELSET] = {
9286                 .call           = nf_tables_delset,
9287                 .type           = NFNL_CB_BATCH,
9288                 .attr_count     = NFTA_SET_MAX,
9289                 .policy         = nft_set_policy,
9290         },
9291         [NFT_MSG_DESTROYSET] = {
9292                 .call           = nf_tables_delset,
9293                 .type           = NFNL_CB_BATCH,
9294                 .attr_count     = NFTA_SET_MAX,
9295                 .policy         = nft_set_policy,
9296         },
9297         [NFT_MSG_NEWSETELEM] = {
9298                 .call           = nf_tables_newsetelem,
9299                 .type           = NFNL_CB_BATCH,
9300                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9301                 .policy         = nft_set_elem_list_policy,
9302         },
9303         [NFT_MSG_GETSETELEM] = {
9304                 .call           = nf_tables_getsetelem,
9305                 .type           = NFNL_CB_RCU,
9306                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9307                 .policy         = nft_set_elem_list_policy,
9308         },
9309         [NFT_MSG_GETSETELEM_RESET] = {
9310                 .call           = nf_tables_getsetelem_reset,
9311                 .type           = NFNL_CB_RCU,
9312                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9313                 .policy         = nft_set_elem_list_policy,
9314         },
9315         [NFT_MSG_DELSETELEM] = {
9316                 .call           = nf_tables_delsetelem,
9317                 .type           = NFNL_CB_BATCH,
9318                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9319                 .policy         = nft_set_elem_list_policy,
9320         },
9321         [NFT_MSG_DESTROYSETELEM] = {
9322                 .call           = nf_tables_delsetelem,
9323                 .type           = NFNL_CB_BATCH,
9324                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9325                 .policy         = nft_set_elem_list_policy,
9326         },
9327         [NFT_MSG_GETGEN] = {
9328                 .call           = nf_tables_getgen,
9329                 .type           = NFNL_CB_RCU,
9330         },
9331         [NFT_MSG_NEWOBJ] = {
9332                 .call           = nf_tables_newobj,
9333                 .type           = NFNL_CB_BATCH,
9334                 .attr_count     = NFTA_OBJ_MAX,
9335                 .policy         = nft_obj_policy,
9336         },
9337         [NFT_MSG_GETOBJ] = {
9338                 .call           = nf_tables_getobj,
9339                 .type           = NFNL_CB_RCU,
9340                 .attr_count     = NFTA_OBJ_MAX,
9341                 .policy         = nft_obj_policy,
9342         },
9343         [NFT_MSG_DELOBJ] = {
9344                 .call           = nf_tables_delobj,
9345                 .type           = NFNL_CB_BATCH,
9346                 .attr_count     = NFTA_OBJ_MAX,
9347                 .policy         = nft_obj_policy,
9348         },
9349         [NFT_MSG_DESTROYOBJ] = {
9350                 .call           = nf_tables_delobj,
9351                 .type           = NFNL_CB_BATCH,
9352                 .attr_count     = NFTA_OBJ_MAX,
9353                 .policy         = nft_obj_policy,
9354         },
9355         [NFT_MSG_GETOBJ_RESET] = {
9356                 .call           = nf_tables_getobj,
9357                 .type           = NFNL_CB_RCU,
9358                 .attr_count     = NFTA_OBJ_MAX,
9359                 .policy         = nft_obj_policy,
9360         },
9361         [NFT_MSG_NEWFLOWTABLE] = {
9362                 .call           = nf_tables_newflowtable,
9363                 .type           = NFNL_CB_BATCH,
9364                 .attr_count     = NFTA_FLOWTABLE_MAX,
9365                 .policy         = nft_flowtable_policy,
9366         },
9367         [NFT_MSG_GETFLOWTABLE] = {
9368                 .call           = nf_tables_getflowtable,
9369                 .type           = NFNL_CB_RCU,
9370                 .attr_count     = NFTA_FLOWTABLE_MAX,
9371                 .policy         = nft_flowtable_policy,
9372         },
9373         [NFT_MSG_DELFLOWTABLE] = {
9374                 .call           = nf_tables_delflowtable,
9375                 .type           = NFNL_CB_BATCH,
9376                 .attr_count     = NFTA_FLOWTABLE_MAX,
9377                 .policy         = nft_flowtable_policy,
9378         },
9379         [NFT_MSG_DESTROYFLOWTABLE] = {
9380                 .call           = nf_tables_delflowtable,
9381                 .type           = NFNL_CB_BATCH,
9382                 .attr_count     = NFTA_FLOWTABLE_MAX,
9383                 .policy         = nft_flowtable_policy,
9384         },
9385 };
9386
9387 static int nf_tables_validate(struct net *net)
9388 {
9389         struct nftables_pernet *nft_net = nft_pernet(net);
9390         struct nft_table *table;
9391
9392         list_for_each_entry(table, &nft_net->tables, list) {
9393                 switch (table->validate_state) {
9394                 case NFT_VALIDATE_SKIP:
9395                         continue;
9396                 case NFT_VALIDATE_NEED:
9397                         nft_validate_state_update(table, NFT_VALIDATE_DO);
9398                         fallthrough;
9399                 case NFT_VALIDATE_DO:
9400                         if (nft_table_validate(net, table) < 0)
9401                                 return -EAGAIN;
9402
9403                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
9404                         break;
9405                 }
9406         }
9407
9408         return 0;
9409 }
9410
9411 /* a drop policy has to be deferred until all rules have been activated,
9412  * otherwise a large ruleset that contains a drop-policy base chain will
9413  * cause all packets to get dropped until the full transaction has been
9414  * processed.
9415  *
9416  * We defer the drop policy until the transaction has been finalized.
9417  */
9418 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
9419 {
9420         struct nft_base_chain *basechain;
9421
9422         if (nft_trans_chain_policy(trans) != NF_DROP)
9423                 return;
9424
9425         if (!nft_is_base_chain(trans->ctx.chain))
9426                 return;
9427
9428         basechain = nft_base_chain(trans->ctx.chain);
9429         basechain->policy = NF_DROP;
9430 }
9431
9432 static void nft_chain_commit_update(struct nft_trans *trans)
9433 {
9434         struct nft_base_chain *basechain;
9435
9436         if (nft_trans_chain_name(trans)) {
9437                 rhltable_remove(&trans->ctx.table->chains_ht,
9438                                 &trans->ctx.chain->rhlhead,
9439                                 nft_chain_ht_params);
9440                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
9441                 rhltable_insert_key(&trans->ctx.table->chains_ht,
9442                                     trans->ctx.chain->name,
9443                                     &trans->ctx.chain->rhlhead,
9444                                     nft_chain_ht_params);
9445         }
9446
9447         if (!nft_is_base_chain(trans->ctx.chain))
9448                 return;
9449
9450         nft_chain_stats_replace(trans);
9451
9452         basechain = nft_base_chain(trans->ctx.chain);
9453
9454         switch (nft_trans_chain_policy(trans)) {
9455         case NF_DROP:
9456         case NF_ACCEPT:
9457                 basechain->policy = nft_trans_chain_policy(trans);
9458                 break;
9459         }
9460 }
9461
9462 static void nft_obj_commit_update(struct nft_trans *trans)
9463 {
9464         struct nft_object *newobj;
9465         struct nft_object *obj;
9466
9467         obj = nft_trans_obj(trans);
9468         newobj = nft_trans_obj_newobj(trans);
9469
9470         if (obj->ops->update)
9471                 obj->ops->update(obj, newobj);
9472
9473         nft_obj_destroy(&trans->ctx, newobj);
9474 }
9475
9476 static void nft_commit_release(struct nft_trans *trans)
9477 {
9478         switch (trans->msg_type) {
9479         case NFT_MSG_DELTABLE:
9480         case NFT_MSG_DESTROYTABLE:
9481                 nf_tables_table_destroy(&trans->ctx);
9482                 break;
9483         case NFT_MSG_NEWCHAIN:
9484                 free_percpu(nft_trans_chain_stats(trans));
9485                 kfree(nft_trans_chain_name(trans));
9486                 break;
9487         case NFT_MSG_DELCHAIN:
9488         case NFT_MSG_DESTROYCHAIN:
9489                 if (nft_trans_chain_update(trans))
9490                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9491                 else
9492                         nf_tables_chain_destroy(&trans->ctx);
9493                 break;
9494         case NFT_MSG_DELRULE:
9495         case NFT_MSG_DESTROYRULE:
9496                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9497                 break;
9498         case NFT_MSG_DELSET:
9499         case NFT_MSG_DESTROYSET:
9500                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9501                 break;
9502         case NFT_MSG_DELSETELEM:
9503         case NFT_MSG_DESTROYSETELEM:
9504                 nf_tables_set_elem_destroy(&trans->ctx,
9505                                            nft_trans_elem_set(trans),
9506                                            nft_trans_elem_priv(trans));
9507                 break;
9508         case NFT_MSG_DELOBJ:
9509         case NFT_MSG_DESTROYOBJ:
9510                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9511                 break;
9512         case NFT_MSG_DELFLOWTABLE:
9513         case NFT_MSG_DESTROYFLOWTABLE:
9514                 if (nft_trans_flowtable_update(trans))
9515                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9516                 else
9517                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9518                 break;
9519         }
9520
9521         if (trans->put_net)
9522                 put_net(trans->ctx.net);
9523
9524         kfree(trans);
9525 }
9526
9527 static void nf_tables_trans_destroy_work(struct work_struct *w)
9528 {
9529         struct nft_trans *trans, *next;
9530         LIST_HEAD(head);
9531
9532         spin_lock(&nf_tables_destroy_list_lock);
9533         list_splice_init(&nf_tables_destroy_list, &head);
9534         spin_unlock(&nf_tables_destroy_list_lock);
9535
9536         if (list_empty(&head))
9537                 return;
9538
9539         synchronize_rcu();
9540
9541         list_for_each_entry_safe(trans, next, &head, list) {
9542                 nft_trans_list_del(trans);
9543                 nft_commit_release(trans);
9544         }
9545 }
9546
9547 void nf_tables_trans_destroy_flush_work(void)
9548 {
9549         flush_work(&trans_destroy_work);
9550 }
9551 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9552
9553 static bool nft_expr_reduce(struct nft_regs_track *track,
9554                             const struct nft_expr *expr)
9555 {
9556         return false;
9557 }
9558
9559 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9560 {
9561         const struct nft_expr *expr, *last;
9562         struct nft_regs_track track = {};
9563         unsigned int size, data_size;
9564         void *data, *data_boundary;
9565         struct nft_rule_dp *prule;
9566         struct nft_rule *rule;
9567
9568         /* already handled or inactive chain? */
9569         if (chain->blob_next || !nft_is_active_next(net, chain))
9570                 return 0;
9571
9572         data_size = 0;
9573         list_for_each_entry(rule, &chain->rules, list) {
9574                 if (nft_is_active_next(net, rule)) {
9575                         data_size += sizeof(*prule) + rule->dlen;
9576                         if (data_size > INT_MAX)
9577                                 return -ENOMEM;
9578                 }
9579         }
9580
9581         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
9582         if (!chain->blob_next)
9583                 return -ENOMEM;
9584
9585         data = (void *)chain->blob_next->data;
9586         data_boundary = data + data_size;
9587         size = 0;
9588
9589         list_for_each_entry(rule, &chain->rules, list) {
9590                 if (!nft_is_active_next(net, rule))
9591                         continue;
9592
9593                 prule = (struct nft_rule_dp *)data;
9594                 data += offsetof(struct nft_rule_dp, data);
9595                 if (WARN_ON_ONCE(data > data_boundary))
9596                         return -ENOMEM;
9597
9598                 size = 0;
9599                 track.last = nft_expr_last(rule);
9600                 nft_rule_for_each_expr(expr, last, rule) {
9601                         track.cur = expr;
9602
9603                         if (nft_expr_reduce(&track, expr)) {
9604                                 expr = track.cur;
9605                                 continue;
9606                         }
9607
9608                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9609                                 return -ENOMEM;
9610
9611                         memcpy(data + size, expr, expr->ops->size);
9612                         size += expr->ops->size;
9613                 }
9614                 if (WARN_ON_ONCE(size >= 1 << 12))
9615                         return -ENOMEM;
9616
9617                 prule->handle = rule->handle;
9618                 prule->dlen = size;
9619                 prule->is_last = 0;
9620
9621                 data += size;
9622                 size = 0;
9623                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9624         }
9625
9626         if (WARN_ON_ONCE(data > data_boundary))
9627                 return -ENOMEM;
9628
9629         prule = (struct nft_rule_dp *)data;
9630         nft_last_rule(chain, prule);
9631
9632         return 0;
9633 }
9634
9635 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9636 {
9637         struct nftables_pernet *nft_net = nft_pernet(net);
9638         struct nft_trans *trans, *next;
9639
9640         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9641                 struct nft_chain *chain = trans->ctx.chain;
9642
9643                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9644                     trans->msg_type == NFT_MSG_DELRULE) {
9645                         kvfree(chain->blob_next);
9646                         chain->blob_next = NULL;
9647                 }
9648         }
9649 }
9650
9651 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9652 {
9653         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9654
9655         kvfree(l->blob);
9656 }
9657
9658 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9659 {
9660         struct nft_rule_dp_last *last;
9661
9662         /* last rule trailer is after end marker */
9663         last = (void *)blob + sizeof(*blob) + blob->size;
9664         last->blob = blob;
9665
9666         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9667 }
9668
9669 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9670 {
9671         struct nft_rule_blob *g0, *g1;
9672         bool next_genbit;
9673
9674         next_genbit = nft_gencursor_next(net);
9675
9676         g0 = rcu_dereference_protected(chain->blob_gen_0,
9677                                        lockdep_commit_lock_is_held(net));
9678         g1 = rcu_dereference_protected(chain->blob_gen_1,
9679                                        lockdep_commit_lock_is_held(net));
9680
9681         /* No changes to this chain? */
9682         if (chain->blob_next == NULL) {
9683                 /* chain had no change in last or next generation */
9684                 if (g0 == g1)
9685                         return;
9686                 /*
9687                  * chain had no change in this generation; make sure next
9688                  * one uses same rules as current generation.
9689                  */
9690                 if (next_genbit) {
9691                         rcu_assign_pointer(chain->blob_gen_1, g0);
9692                         nf_tables_commit_chain_free_rules_old(g1);
9693                 } else {
9694                         rcu_assign_pointer(chain->blob_gen_0, g1);
9695                         nf_tables_commit_chain_free_rules_old(g0);
9696                 }
9697
9698                 return;
9699         }
9700
9701         if (next_genbit)
9702                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9703         else
9704                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9705
9706         chain->blob_next = NULL;
9707
9708         if (g0 == g1)
9709                 return;
9710
9711         if (next_genbit)
9712                 nf_tables_commit_chain_free_rules_old(g1);
9713         else
9714                 nf_tables_commit_chain_free_rules_old(g0);
9715 }
9716
9717 static void nft_obj_del(struct nft_object *obj)
9718 {
9719         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9720         list_del_rcu(&obj->list);
9721 }
9722
9723 void nft_chain_del(struct nft_chain *chain)
9724 {
9725         struct nft_table *table = chain->table;
9726
9727         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9728                                      nft_chain_ht_params));
9729         list_del_rcu(&chain->list);
9730 }
9731
9732 static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
9733                                         struct nft_trans_gc *trans)
9734 {
9735         struct nft_elem_priv **priv = trans->priv;
9736         unsigned int i;
9737
9738         for (i = 0; i < trans->count; i++) {
9739                 nft_setelem_data_deactivate(ctx->net, trans->set, priv[i]);
9740                 nft_setelem_remove(ctx->net, trans->set, priv[i]);
9741         }
9742 }
9743
9744 void nft_trans_gc_destroy(struct nft_trans_gc *trans)
9745 {
9746         nft_set_put(trans->set);
9747         put_net(trans->net);
9748         kfree(trans);
9749 }
9750
9751 static void nft_trans_gc_trans_free(struct rcu_head *rcu)
9752 {
9753         struct nft_elem_priv *elem_priv;
9754         struct nft_trans_gc *trans;
9755         struct nft_ctx ctx = {};
9756         unsigned int i;
9757
9758         trans = container_of(rcu, struct nft_trans_gc, rcu);
9759         ctx.net = read_pnet(&trans->set->net);
9760
9761         for (i = 0; i < trans->count; i++) {
9762                 elem_priv = trans->priv[i];
9763                 if (!nft_setelem_is_catchall(trans->set, elem_priv))
9764                         atomic_dec(&trans->set->nelems);
9765
9766                 nf_tables_set_elem_destroy(&ctx, trans->set, elem_priv);
9767         }
9768
9769         nft_trans_gc_destroy(trans);
9770 }
9771
9772 static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
9773 {
9774         struct nftables_pernet *nft_net;
9775         struct nft_ctx ctx = {};
9776
9777         nft_net = nft_pernet(trans->net);
9778
9779         mutex_lock(&nft_net->commit_mutex);
9780
9781         /* Check for race with transaction, otherwise this batch refers to
9782          * stale objects that might not be there anymore. Skip transaction if
9783          * set has been destroyed from control plane transaction in case gc
9784          * worker loses race.
9785          */
9786         if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
9787                 mutex_unlock(&nft_net->commit_mutex);
9788                 return false;
9789         }
9790
9791         ctx.net = trans->net;
9792         ctx.table = trans->set->table;
9793
9794         nft_trans_gc_setelem_remove(&ctx, trans);
9795         mutex_unlock(&nft_net->commit_mutex);
9796
9797         return true;
9798 }
9799
9800 static void nft_trans_gc_work(struct work_struct *work)
9801 {
9802         struct nft_trans_gc *trans, *next;
9803         LIST_HEAD(trans_gc_list);
9804
9805         spin_lock(&nf_tables_gc_list_lock);
9806         list_splice_init(&nf_tables_gc_list, &trans_gc_list);
9807         spin_unlock(&nf_tables_gc_list_lock);
9808
9809         list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
9810                 list_del(&trans->list);
9811                 if (!nft_trans_gc_work_done(trans)) {
9812                         nft_trans_gc_destroy(trans);
9813                         continue;
9814                 }
9815                 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9816         }
9817 }
9818
9819 struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
9820                                         unsigned int gc_seq, gfp_t gfp)
9821 {
9822         struct net *net = read_pnet(&set->net);
9823         struct nft_trans_gc *trans;
9824
9825         trans = kzalloc(sizeof(*trans), gfp);
9826         if (!trans)
9827                 return NULL;
9828
9829         trans->net = maybe_get_net(net);
9830         if (!trans->net) {
9831                 kfree(trans);
9832                 return NULL;
9833         }
9834
9835         refcount_inc(&set->refs);
9836         trans->set = set;
9837         trans->seq = gc_seq;
9838
9839         return trans;
9840 }
9841
9842 void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
9843 {
9844         trans->priv[trans->count++] = priv;
9845 }
9846
9847 static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
9848 {
9849         spin_lock(&nf_tables_gc_list_lock);
9850         list_add_tail(&trans->list, &nf_tables_gc_list);
9851         spin_unlock(&nf_tables_gc_list_lock);
9852
9853         schedule_work(&trans_gc_work);
9854 }
9855
9856 static int nft_trans_gc_space(struct nft_trans_gc *trans)
9857 {
9858         return NFT_TRANS_GC_BATCHCOUNT - trans->count;
9859 }
9860
9861 struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
9862                                               unsigned int gc_seq, gfp_t gfp)
9863 {
9864         struct nft_set *set;
9865
9866         if (nft_trans_gc_space(gc))
9867                 return gc;
9868
9869         set = gc->set;
9870         nft_trans_gc_queue_work(gc);
9871
9872         return nft_trans_gc_alloc(set, gc_seq, gfp);
9873 }
9874
9875 void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
9876 {
9877         if (trans->count == 0) {
9878                 nft_trans_gc_destroy(trans);
9879                 return;
9880         }
9881
9882         nft_trans_gc_queue_work(trans);
9883 }
9884
9885 struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
9886 {
9887         struct nft_set *set;
9888
9889         if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
9890                 return NULL;
9891
9892         if (nft_trans_gc_space(gc))
9893                 return gc;
9894
9895         set = gc->set;
9896         call_rcu(&gc->rcu, nft_trans_gc_trans_free);
9897
9898         return nft_trans_gc_alloc(set, 0, gfp);
9899 }
9900
9901 void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
9902 {
9903         WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
9904
9905         if (trans->count == 0) {
9906                 nft_trans_gc_destroy(trans);
9907                 return;
9908         }
9909
9910         call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9911 }
9912
9913 struct nft_trans_gc *nft_trans_gc_catchall_async(struct nft_trans_gc *gc,
9914                                                  unsigned int gc_seq)
9915 {
9916         struct nft_set_elem_catchall *catchall;
9917         const struct nft_set *set = gc->set;
9918         struct nft_set_ext *ext;
9919
9920         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9921                 ext = nft_set_elem_ext(set, catchall->elem);
9922
9923                 if (!nft_set_elem_expired(ext))
9924                         continue;
9925                 if (nft_set_elem_is_dead(ext))
9926                         goto dead_elem;
9927
9928                 nft_set_elem_dead(ext);
9929 dead_elem:
9930                 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
9931                 if (!gc)
9932                         return NULL;
9933
9934                 nft_trans_gc_elem_add(gc, catchall->elem);
9935         }
9936
9937         return gc;
9938 }
9939
9940 struct nft_trans_gc *nft_trans_gc_catchall_sync(struct nft_trans_gc *gc)
9941 {
9942         struct nft_set_elem_catchall *catchall, *next;
9943         u64 tstamp = nft_net_tstamp(gc->net);
9944         const struct nft_set *set = gc->set;
9945         struct nft_elem_priv *elem_priv;
9946         struct nft_set_ext *ext;
9947
9948         WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net));
9949
9950         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
9951                 ext = nft_set_elem_ext(set, catchall->elem);
9952
9953                 if (!__nft_set_elem_expired(ext, tstamp))
9954                         continue;
9955
9956                 gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL);
9957                 if (!gc)
9958                         return NULL;
9959
9960                 elem_priv = catchall->elem;
9961                 nft_setelem_data_deactivate(gc->net, gc->set, elem_priv);
9962                 nft_setelem_catchall_destroy(catchall);
9963                 nft_trans_gc_elem_add(gc, elem_priv);
9964         }
9965
9966         return gc;
9967 }
9968
9969 static void nf_tables_module_autoload_cleanup(struct net *net)
9970 {
9971         struct nftables_pernet *nft_net = nft_pernet(net);
9972         struct nft_module_request *req, *next;
9973
9974         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9975         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9976                 WARN_ON_ONCE(!req->done);
9977                 list_del(&req->list);
9978                 kfree(req);
9979         }
9980 }
9981
9982 static void nf_tables_commit_release(struct net *net)
9983 {
9984         struct nftables_pernet *nft_net = nft_pernet(net);
9985         struct nft_trans *trans;
9986
9987         /* all side effects have to be made visible.
9988          * For example, if a chain named 'foo' has been deleted, a
9989          * new transaction must not find it anymore.
9990          *
9991          * Memory reclaim happens asynchronously from work queue
9992          * to prevent expensive synchronize_rcu() in commit phase.
9993          */
9994         if (list_empty(&nft_net->commit_list)) {
9995                 nf_tables_module_autoload_cleanup(net);
9996                 mutex_unlock(&nft_net->commit_mutex);
9997                 return;
9998         }
9999
10000         trans = list_last_entry(&nft_net->commit_list,
10001                                 struct nft_trans, list);
10002         get_net(trans->ctx.net);
10003         WARN_ON_ONCE(trans->put_net);
10004
10005         trans->put_net = true;
10006         spin_lock(&nf_tables_destroy_list_lock);
10007         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
10008         spin_unlock(&nf_tables_destroy_list_lock);
10009
10010         nf_tables_module_autoload_cleanup(net);
10011         schedule_work(&trans_destroy_work);
10012
10013         mutex_unlock(&nft_net->commit_mutex);
10014 }
10015
10016 static void nft_commit_notify(struct net *net, u32 portid)
10017 {
10018         struct nftables_pernet *nft_net = nft_pernet(net);
10019         struct sk_buff *batch_skb = NULL, *nskb, *skb;
10020         unsigned char *data;
10021         int len;
10022
10023         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
10024                 if (!batch_skb) {
10025 new_batch:
10026                         batch_skb = skb;
10027                         len = NLMSG_GOODSIZE - skb->len;
10028                         list_del(&skb->list);
10029                         continue;
10030                 }
10031                 len -= skb->len;
10032                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
10033                         data = skb_put(batch_skb, skb->len);
10034                         memcpy(data, skb->data, skb->len);
10035                         list_del(&skb->list);
10036                         kfree_skb(skb);
10037                         continue;
10038                 }
10039                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10040                                NFT_CB(batch_skb).report, GFP_KERNEL);
10041                 goto new_batch;
10042         }
10043
10044         if (batch_skb) {
10045                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10046                                NFT_CB(batch_skb).report, GFP_KERNEL);
10047         }
10048
10049         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10050 }
10051
10052 static int nf_tables_commit_audit_alloc(struct list_head *adl,
10053                                         struct nft_table *table)
10054 {
10055         struct nft_audit_data *adp;
10056
10057         list_for_each_entry(adp, adl, list) {
10058                 if (adp->table == table)
10059                         return 0;
10060         }
10061         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
10062         if (!adp)
10063                 return -ENOMEM;
10064         adp->table = table;
10065         list_add(&adp->list, adl);
10066         return 0;
10067 }
10068
10069 static void nf_tables_commit_audit_free(struct list_head *adl)
10070 {
10071         struct nft_audit_data *adp, *adn;
10072
10073         list_for_each_entry_safe(adp, adn, adl, list) {
10074                 list_del(&adp->list);
10075                 kfree(adp);
10076         }
10077 }
10078
10079 static void nf_tables_commit_audit_collect(struct list_head *adl,
10080                                            struct nft_table *table, u32 op)
10081 {
10082         struct nft_audit_data *adp;
10083
10084         list_for_each_entry(adp, adl, list) {
10085                 if (adp->table == table)
10086                         goto found;
10087         }
10088         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
10089         return;
10090 found:
10091         adp->entries++;
10092         if (!adp->op || adp->op > op)
10093                 adp->op = op;
10094 }
10095
10096 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
10097
10098 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
10099 {
10100         struct nft_audit_data *adp, *adn;
10101         char aubuf[AUNFTABLENAMELEN];
10102
10103         list_for_each_entry_safe(adp, adn, adl, list) {
10104                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
10105                          generation);
10106                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
10107                                 nft2audit_op[adp->op], GFP_KERNEL);
10108                 list_del(&adp->list);
10109                 kfree(adp);
10110         }
10111 }
10112
10113 static void nft_set_commit_update(struct list_head *set_update_list)
10114 {
10115         struct nft_set *set, *next;
10116
10117         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10118                 list_del_init(&set->pending_update);
10119
10120                 if (!set->ops->commit || set->dead)
10121                         continue;
10122
10123                 set->ops->commit(set);
10124         }
10125 }
10126
10127 static unsigned int nft_gc_seq_begin(struct nftables_pernet *nft_net)
10128 {
10129         unsigned int gc_seq;
10130
10131         /* Bump gc counter, it becomes odd, this is the busy mark. */
10132         gc_seq = READ_ONCE(nft_net->gc_seq);
10133         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10134
10135         return gc_seq;
10136 }
10137
10138 static void nft_gc_seq_end(struct nftables_pernet *nft_net, unsigned int gc_seq)
10139 {
10140         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10141 }
10142
10143 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
10144 {
10145         struct nftables_pernet *nft_net = nft_pernet(net);
10146         struct nft_trans *trans, *next;
10147         unsigned int base_seq, gc_seq;
10148         LIST_HEAD(set_update_list);
10149         struct nft_trans_elem *te;
10150         struct nft_chain *chain;
10151         struct nft_table *table;
10152         LIST_HEAD(adl);
10153         int err;
10154
10155         if (list_empty(&nft_net->commit_list)) {
10156                 mutex_unlock(&nft_net->commit_mutex);
10157                 return 0;
10158         }
10159
10160         list_for_each_entry(trans, &nft_net->binding_list, binding_list) {
10161                 switch (trans->msg_type) {
10162                 case NFT_MSG_NEWSET:
10163                         if (!nft_trans_set_update(trans) &&
10164                             nft_set_is_anonymous(nft_trans_set(trans)) &&
10165                             !nft_trans_set_bound(trans)) {
10166                                 pr_warn_once("nftables ruleset with unbound set\n");
10167                                 return -EINVAL;
10168                         }
10169                         break;
10170                 case NFT_MSG_NEWCHAIN:
10171                         if (!nft_trans_chain_update(trans) &&
10172                             nft_chain_binding(nft_trans_chain(trans)) &&
10173                             !nft_trans_chain_bound(trans)) {
10174                                 pr_warn_once("nftables ruleset with unbound chain\n");
10175                                 return -EINVAL;
10176                         }
10177                         break;
10178                 }
10179         }
10180
10181         /* 0. Validate ruleset, otherwise roll back for error reporting. */
10182         if (nf_tables_validate(net) < 0) {
10183                 nft_net->validate_state = NFT_VALIDATE_DO;
10184                 return -EAGAIN;
10185         }
10186
10187         err = nft_flow_rule_offload_commit(net);
10188         if (err < 0)
10189                 return err;
10190
10191         /* 1.  Allocate space for next generation rules_gen_X[] */
10192         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10193                 int ret;
10194
10195                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
10196                 if (ret) {
10197                         nf_tables_commit_chain_prepare_cancel(net);
10198                         nf_tables_commit_audit_free(&adl);
10199                         return ret;
10200                 }
10201                 if (trans->msg_type == NFT_MSG_NEWRULE ||
10202                     trans->msg_type == NFT_MSG_DELRULE) {
10203                         chain = trans->ctx.chain;
10204
10205                         ret = nf_tables_commit_chain_prepare(net, chain);
10206                         if (ret < 0) {
10207                                 nf_tables_commit_chain_prepare_cancel(net);
10208                                 nf_tables_commit_audit_free(&adl);
10209                                 return ret;
10210                         }
10211                 }
10212         }
10213
10214         /* step 2.  Make rules_gen_X visible to packet path */
10215         list_for_each_entry(table, &nft_net->tables, list) {
10216                 list_for_each_entry(chain, &table->chains, list)
10217                         nf_tables_commit_chain(net, chain);
10218         }
10219
10220         /*
10221          * Bump generation counter, invalidate any dump in progress.
10222          * Cannot fail after this point.
10223          */
10224         base_seq = READ_ONCE(nft_net->base_seq);
10225         while (++base_seq == 0)
10226                 ;
10227
10228         WRITE_ONCE(nft_net->base_seq, base_seq);
10229
10230         gc_seq = nft_gc_seq_begin(nft_net);
10231
10232         /* step 3. Start new generation, rules_gen_X now in use. */
10233         net->nft.gencursor = nft_gencursor_next(net);
10234
10235         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10236                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
10237                                                trans->msg_type);
10238                 switch (trans->msg_type) {
10239                 case NFT_MSG_NEWTABLE:
10240                         if (nft_trans_table_update(trans)) {
10241                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10242                                         nft_trans_destroy(trans);
10243                                         break;
10244                                 }
10245                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
10246                                         nf_tables_table_disable(net, trans->ctx.table);
10247
10248                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10249                         } else {
10250                                 nft_clear(net, trans->ctx.table);
10251                         }
10252                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
10253                         nft_trans_destroy(trans);
10254                         break;
10255                 case NFT_MSG_DELTABLE:
10256                 case NFT_MSG_DESTROYTABLE:
10257                         list_del_rcu(&trans->ctx.table->list);
10258                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
10259                         break;
10260                 case NFT_MSG_NEWCHAIN:
10261                         if (nft_trans_chain_update(trans)) {
10262                                 nft_chain_commit_update(trans);
10263                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
10264                                                        &nft_trans_chain_hooks(trans));
10265                                 list_splice(&nft_trans_chain_hooks(trans),
10266                                             &nft_trans_basechain(trans)->hook_list);
10267                                 /* trans destroyed after rcu grace period */
10268                         } else {
10269                                 nft_chain_commit_drop_policy(trans);
10270                                 nft_clear(net, trans->ctx.chain);
10271                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
10272                                 nft_trans_destroy(trans);
10273                         }
10274                         break;
10275                 case NFT_MSG_DELCHAIN:
10276                 case NFT_MSG_DESTROYCHAIN:
10277                         if (nft_trans_chain_update(trans)) {
10278                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10279                                                        &nft_trans_chain_hooks(trans));
10280                                 if (!(trans->ctx.table->flags & NFT_TABLE_F_DORMANT)) {
10281                                         nft_netdev_unregister_hooks(net,
10282                                                                     &nft_trans_chain_hooks(trans),
10283                                                                     true);
10284                                 }
10285                         } else {
10286                                 nft_chain_del(trans->ctx.chain);
10287                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10288                                                        NULL);
10289                                 nf_tables_unregister_hook(trans->ctx.net,
10290                                                           trans->ctx.table,
10291                                                           trans->ctx.chain);
10292                         }
10293                         break;
10294                 case NFT_MSG_NEWRULE:
10295                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10296                         nf_tables_rule_notify(&trans->ctx,
10297                                               nft_trans_rule(trans),
10298                                               NFT_MSG_NEWRULE);
10299                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10300                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10301
10302                         nft_trans_destroy(trans);
10303                         break;
10304                 case NFT_MSG_DELRULE:
10305                 case NFT_MSG_DESTROYRULE:
10306                         list_del_rcu(&nft_trans_rule(trans)->list);
10307                         nf_tables_rule_notify(&trans->ctx,
10308                                               nft_trans_rule(trans),
10309                                               trans->msg_type);
10310                         nft_rule_expr_deactivate(&trans->ctx,
10311                                                  nft_trans_rule(trans),
10312                                                  NFT_TRANS_COMMIT);
10313
10314                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10315                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10316                         break;
10317                 case NFT_MSG_NEWSET:
10318                         if (nft_trans_set_update(trans)) {
10319                                 struct nft_set *set = nft_trans_set(trans);
10320
10321                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
10322                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
10323
10324                                 if (nft_trans_set_size(trans))
10325                                         WRITE_ONCE(set->size, nft_trans_set_size(trans));
10326                         } else {
10327                                 nft_clear(net, nft_trans_set(trans));
10328                                 /* This avoids hitting -EBUSY when deleting the table
10329                                  * from the transaction.
10330                                  */
10331                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
10332                                     !list_empty(&nft_trans_set(trans)->bindings))
10333                                         nft_use_dec(&trans->ctx.table->use);
10334                         }
10335                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10336                                              NFT_MSG_NEWSET, GFP_KERNEL);
10337                         nft_trans_destroy(trans);
10338                         break;
10339                 case NFT_MSG_DELSET:
10340                 case NFT_MSG_DESTROYSET:
10341                         nft_trans_set(trans)->dead = 1;
10342                         list_del_rcu(&nft_trans_set(trans)->list);
10343                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10344                                              trans->msg_type, GFP_KERNEL);
10345                         break;
10346                 case NFT_MSG_NEWSETELEM:
10347                         te = (struct nft_trans_elem *)trans->data;
10348
10349                         nft_setelem_activate(net, te->set, te->elem_priv);
10350                         nf_tables_setelem_notify(&trans->ctx, te->set,
10351                                                  te->elem_priv,
10352                                                  NFT_MSG_NEWSETELEM);
10353                         if (te->set->ops->commit &&
10354                             list_empty(&te->set->pending_update)) {
10355                                 list_add_tail(&te->set->pending_update,
10356                                               &set_update_list);
10357                         }
10358                         nft_trans_destroy(trans);
10359                         break;
10360                 case NFT_MSG_DELSETELEM:
10361                 case NFT_MSG_DESTROYSETELEM:
10362                         te = (struct nft_trans_elem *)trans->data;
10363
10364                         nf_tables_setelem_notify(&trans->ctx, te->set,
10365                                                  te->elem_priv,
10366                                                  trans->msg_type);
10367                         nft_setelem_remove(net, te->set, te->elem_priv);
10368                         if (!nft_setelem_is_catchall(te->set, te->elem_priv)) {
10369                                 atomic_dec(&te->set->nelems);
10370                                 te->set->ndeact--;
10371                         }
10372                         if (te->set->ops->commit &&
10373                             list_empty(&te->set->pending_update)) {
10374                                 list_add_tail(&te->set->pending_update,
10375                                               &set_update_list);
10376                         }
10377                         break;
10378                 case NFT_MSG_NEWOBJ:
10379                         if (nft_trans_obj_update(trans)) {
10380                                 nft_obj_commit_update(trans);
10381                                 nf_tables_obj_notify(&trans->ctx,
10382                                                      nft_trans_obj(trans),
10383                                                      NFT_MSG_NEWOBJ);
10384                         } else {
10385                                 nft_clear(net, nft_trans_obj(trans));
10386                                 nf_tables_obj_notify(&trans->ctx,
10387                                                      nft_trans_obj(trans),
10388                                                      NFT_MSG_NEWOBJ);
10389                                 nft_trans_destroy(trans);
10390                         }
10391                         break;
10392                 case NFT_MSG_DELOBJ:
10393                 case NFT_MSG_DESTROYOBJ:
10394                         nft_obj_del(nft_trans_obj(trans));
10395                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
10396                                              trans->msg_type);
10397                         break;
10398                 case NFT_MSG_NEWFLOWTABLE:
10399                         if (nft_trans_flowtable_update(trans)) {
10400                                 nft_trans_flowtable(trans)->data.flags =
10401                                         nft_trans_flowtable_flags(trans);
10402                                 nf_tables_flowtable_notify(&trans->ctx,
10403                                                            nft_trans_flowtable(trans),
10404                                                            &nft_trans_flowtable_hooks(trans),
10405                                                            NFT_MSG_NEWFLOWTABLE);
10406                                 list_splice(&nft_trans_flowtable_hooks(trans),
10407                                             &nft_trans_flowtable(trans)->hook_list);
10408                         } else {
10409                                 nft_clear(net, nft_trans_flowtable(trans));
10410                                 nf_tables_flowtable_notify(&trans->ctx,
10411                                                            nft_trans_flowtable(trans),
10412                                                            NULL,
10413                                                            NFT_MSG_NEWFLOWTABLE);
10414                         }
10415                         nft_trans_destroy(trans);
10416                         break;
10417                 case NFT_MSG_DELFLOWTABLE:
10418                 case NFT_MSG_DESTROYFLOWTABLE:
10419                         if (nft_trans_flowtable_update(trans)) {
10420                                 nf_tables_flowtable_notify(&trans->ctx,
10421                                                            nft_trans_flowtable(trans),
10422                                                            &nft_trans_flowtable_hooks(trans),
10423                                                            trans->msg_type);
10424                                 nft_unregister_flowtable_net_hooks(net,
10425                                                                    &nft_trans_flowtable_hooks(trans));
10426                         } else {
10427                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10428                                 nf_tables_flowtable_notify(&trans->ctx,
10429                                                            nft_trans_flowtable(trans),
10430                                                            NULL,
10431                                                            trans->msg_type);
10432                                 nft_unregister_flowtable_net_hooks(net,
10433                                                 &nft_trans_flowtable(trans)->hook_list);
10434                         }
10435                         break;
10436                 }
10437         }
10438
10439         nft_set_commit_update(&set_update_list);
10440
10441         nft_commit_notify(net, NETLINK_CB(skb).portid);
10442         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
10443         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10444
10445         nft_gc_seq_end(nft_net, gc_seq);
10446         nft_net->validate_state = NFT_VALIDATE_SKIP;
10447         nf_tables_commit_release(net);
10448
10449         return 0;
10450 }
10451
10452 static void nf_tables_module_autoload(struct net *net)
10453 {
10454         struct nftables_pernet *nft_net = nft_pernet(net);
10455         struct nft_module_request *req, *next;
10456         LIST_HEAD(module_list);
10457
10458         list_splice_init(&nft_net->module_list, &module_list);
10459         mutex_unlock(&nft_net->commit_mutex);
10460         list_for_each_entry_safe(req, next, &module_list, list) {
10461                 request_module("%s", req->module);
10462                 req->done = true;
10463         }
10464         mutex_lock(&nft_net->commit_mutex);
10465         list_splice(&module_list, &nft_net->module_list);
10466 }
10467
10468 static void nf_tables_abort_release(struct nft_trans *trans)
10469 {
10470         switch (trans->msg_type) {
10471         case NFT_MSG_NEWTABLE:
10472                 nf_tables_table_destroy(&trans->ctx);
10473                 break;
10474         case NFT_MSG_NEWCHAIN:
10475                 if (nft_trans_chain_update(trans))
10476                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
10477                 else
10478                         nf_tables_chain_destroy(&trans->ctx);
10479                 break;
10480         case NFT_MSG_NEWRULE:
10481                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
10482                 break;
10483         case NFT_MSG_NEWSET:
10484                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
10485                 break;
10486         case NFT_MSG_NEWSETELEM:
10487                 nft_set_elem_destroy(nft_trans_elem_set(trans),
10488                                      nft_trans_elem_priv(trans), true);
10489                 break;
10490         case NFT_MSG_NEWOBJ:
10491                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
10492                 break;
10493         case NFT_MSG_NEWFLOWTABLE:
10494                 if (nft_trans_flowtable_update(trans))
10495                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
10496                 else
10497                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
10498                 break;
10499         }
10500         kfree(trans);
10501 }
10502
10503 static void nft_set_abort_update(struct list_head *set_update_list)
10504 {
10505         struct nft_set *set, *next;
10506
10507         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10508                 list_del_init(&set->pending_update);
10509
10510                 if (!set->ops->abort)
10511                         continue;
10512
10513                 set->ops->abort(set);
10514         }
10515 }
10516
10517 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
10518 {
10519         struct nftables_pernet *nft_net = nft_pernet(net);
10520         struct nft_trans *trans, *next;
10521         LIST_HEAD(set_update_list);
10522         struct nft_trans_elem *te;
10523         int err = 0;
10524
10525         if (action == NFNL_ABORT_VALIDATE &&
10526             nf_tables_validate(net) < 0)
10527                 err = -EAGAIN;
10528
10529         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
10530                                          list) {
10531                 switch (trans->msg_type) {
10532                 case NFT_MSG_NEWTABLE:
10533                         if (nft_trans_table_update(trans)) {
10534                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10535                                         nft_trans_destroy(trans);
10536                                         break;
10537                                 }
10538                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
10539                                         nf_tables_table_disable(net, trans->ctx.table);
10540                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
10541                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
10542                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
10543                                 }
10544                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_ORPHAN) {
10545                                         trans->ctx.table->flags &= ~NFT_TABLE_F_OWNER;
10546                                         trans->ctx.table->nlpid = 0;
10547                                 }
10548                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10549                                 nft_trans_destroy(trans);
10550                         } else {
10551                                 list_del_rcu(&trans->ctx.table->list);
10552                         }
10553                         break;
10554                 case NFT_MSG_DELTABLE:
10555                 case NFT_MSG_DESTROYTABLE:
10556                         nft_clear(trans->ctx.net, trans->ctx.table);
10557                         nft_trans_destroy(trans);
10558                         break;
10559                 case NFT_MSG_NEWCHAIN:
10560                         if (nft_trans_chain_update(trans)) {
10561                                 if (!(trans->ctx.table->flags & NFT_TABLE_F_DORMANT)) {
10562                                         nft_netdev_unregister_hooks(net,
10563                                                                     &nft_trans_chain_hooks(trans),
10564                                                                     true);
10565                                 }
10566                                 free_percpu(nft_trans_chain_stats(trans));
10567                                 kfree(nft_trans_chain_name(trans));
10568                                 nft_trans_destroy(trans);
10569                         } else {
10570                                 if (nft_trans_chain_bound(trans)) {
10571                                         nft_trans_destroy(trans);
10572                                         break;
10573                                 }
10574                                 nft_use_dec_restore(&trans->ctx.table->use);
10575                                 nft_chain_del(trans->ctx.chain);
10576                                 nf_tables_unregister_hook(trans->ctx.net,
10577                                                           trans->ctx.table,
10578                                                           trans->ctx.chain);
10579                         }
10580                         break;
10581                 case NFT_MSG_DELCHAIN:
10582                 case NFT_MSG_DESTROYCHAIN:
10583                         if (nft_trans_chain_update(trans)) {
10584                                 list_splice(&nft_trans_chain_hooks(trans),
10585                                             &nft_trans_basechain(trans)->hook_list);
10586                         } else {
10587                                 nft_use_inc_restore(&trans->ctx.table->use);
10588                                 nft_clear(trans->ctx.net, trans->ctx.chain);
10589                         }
10590                         nft_trans_destroy(trans);
10591                         break;
10592                 case NFT_MSG_NEWRULE:
10593                         if (nft_trans_rule_bound(trans)) {
10594                                 nft_trans_destroy(trans);
10595                                 break;
10596                         }
10597                         nft_use_dec_restore(&trans->ctx.chain->use);
10598                         list_del_rcu(&nft_trans_rule(trans)->list);
10599                         nft_rule_expr_deactivate(&trans->ctx,
10600                                                  nft_trans_rule(trans),
10601                                                  NFT_TRANS_ABORT);
10602                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10603                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10604                         break;
10605                 case NFT_MSG_DELRULE:
10606                 case NFT_MSG_DESTROYRULE:
10607                         nft_use_inc_restore(&trans->ctx.chain->use);
10608                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10609                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
10610                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10611                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10612
10613                         nft_trans_destroy(trans);
10614                         break;
10615                 case NFT_MSG_NEWSET:
10616                         if (nft_trans_set_update(trans)) {
10617                                 nft_trans_destroy(trans);
10618                                 break;
10619                         }
10620                         nft_use_dec_restore(&trans->ctx.table->use);
10621                         if (nft_trans_set_bound(trans)) {
10622                                 nft_trans_destroy(trans);
10623                                 break;
10624                         }
10625                         nft_trans_set(trans)->dead = 1;
10626                         list_del_rcu(&nft_trans_set(trans)->list);
10627                         break;
10628                 case NFT_MSG_DELSET:
10629                 case NFT_MSG_DESTROYSET:
10630                         nft_use_inc_restore(&trans->ctx.table->use);
10631                         nft_clear(trans->ctx.net, nft_trans_set(trans));
10632                         if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
10633                                 nft_map_activate(&trans->ctx, nft_trans_set(trans));
10634
10635                         nft_trans_destroy(trans);
10636                         break;
10637                 case NFT_MSG_NEWSETELEM:
10638                         if (nft_trans_elem_set_bound(trans)) {
10639                                 nft_trans_destroy(trans);
10640                                 break;
10641                         }
10642                         te = (struct nft_trans_elem *)trans->data;
10643                         nft_setelem_remove(net, te->set, te->elem_priv);
10644                         if (!nft_setelem_is_catchall(te->set, te->elem_priv))
10645                                 atomic_dec(&te->set->nelems);
10646
10647                         if (te->set->ops->abort &&
10648                             list_empty(&te->set->pending_update)) {
10649                                 list_add_tail(&te->set->pending_update,
10650                                               &set_update_list);
10651                         }
10652                         break;
10653                 case NFT_MSG_DELSETELEM:
10654                 case NFT_MSG_DESTROYSETELEM:
10655                         te = (struct nft_trans_elem *)trans->data;
10656
10657                         if (!nft_setelem_active_next(net, te->set, te->elem_priv)) {
10658                                 nft_setelem_data_activate(net, te->set, te->elem_priv);
10659                                 nft_setelem_activate(net, te->set, te->elem_priv);
10660                         }
10661                         if (!nft_setelem_is_catchall(te->set, te->elem_priv))
10662                                 te->set->ndeact--;
10663
10664                         if (te->set->ops->abort &&
10665                             list_empty(&te->set->pending_update)) {
10666                                 list_add_tail(&te->set->pending_update,
10667                                               &set_update_list);
10668                         }
10669                         nft_trans_destroy(trans);
10670                         break;
10671                 case NFT_MSG_NEWOBJ:
10672                         if (nft_trans_obj_update(trans)) {
10673                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
10674                                 nft_trans_destroy(trans);
10675                         } else {
10676                                 nft_use_dec_restore(&trans->ctx.table->use);
10677                                 nft_obj_del(nft_trans_obj(trans));
10678                         }
10679                         break;
10680                 case NFT_MSG_DELOBJ:
10681                 case NFT_MSG_DESTROYOBJ:
10682                         nft_use_inc_restore(&trans->ctx.table->use);
10683                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
10684                         nft_trans_destroy(trans);
10685                         break;
10686                 case NFT_MSG_NEWFLOWTABLE:
10687                         if (nft_trans_flowtable_update(trans)) {
10688                                 nft_unregister_flowtable_net_hooks(net,
10689                                                 &nft_trans_flowtable_hooks(trans));
10690                         } else {
10691                                 nft_use_dec_restore(&trans->ctx.table->use);
10692                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10693                                 nft_unregister_flowtable_net_hooks(net,
10694                                                 &nft_trans_flowtable(trans)->hook_list);
10695                         }
10696                         break;
10697                 case NFT_MSG_DELFLOWTABLE:
10698                 case NFT_MSG_DESTROYFLOWTABLE:
10699                         if (nft_trans_flowtable_update(trans)) {
10700                                 list_splice(&nft_trans_flowtable_hooks(trans),
10701                                             &nft_trans_flowtable(trans)->hook_list);
10702                         } else {
10703                                 nft_use_inc_restore(&trans->ctx.table->use);
10704                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
10705                         }
10706                         nft_trans_destroy(trans);
10707                         break;
10708                 }
10709         }
10710
10711         nft_set_abort_update(&set_update_list);
10712
10713         synchronize_rcu();
10714
10715         list_for_each_entry_safe_reverse(trans, next,
10716                                          &nft_net->commit_list, list) {
10717                 nft_trans_list_del(trans);
10718                 nf_tables_abort_release(trans);
10719         }
10720
10721         return err;
10722 }
10723
10724 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
10725                            enum nfnl_abort_action action)
10726 {
10727         struct nftables_pernet *nft_net = nft_pernet(net);
10728         unsigned int gc_seq;
10729         int ret;
10730
10731         gc_seq = nft_gc_seq_begin(nft_net);
10732         ret = __nf_tables_abort(net, action);
10733         nft_gc_seq_end(nft_net, gc_seq);
10734
10735         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
10736
10737         /* module autoload needs to happen after GC sequence update because it
10738          * temporarily releases and grabs mutex again.
10739          */
10740         if (action == NFNL_ABORT_AUTOLOAD)
10741                 nf_tables_module_autoload(net);
10742         else
10743                 nf_tables_module_autoload_cleanup(net);
10744
10745         mutex_unlock(&nft_net->commit_mutex);
10746
10747         return ret;
10748 }
10749
10750 static bool nf_tables_valid_genid(struct net *net, u32 genid)
10751 {
10752         struct nftables_pernet *nft_net = nft_pernet(net);
10753         bool genid_ok;
10754
10755         mutex_lock(&nft_net->commit_mutex);
10756         nft_net->tstamp = get_jiffies_64();
10757
10758         genid_ok = genid == 0 || nft_net->base_seq == genid;
10759         if (!genid_ok)
10760                 mutex_unlock(&nft_net->commit_mutex);
10761
10762         /* else, commit mutex has to be released by commit or abort function */
10763         return genid_ok;
10764 }
10765
10766 static const struct nfnetlink_subsystem nf_tables_subsys = {
10767         .name           = "nf_tables",
10768         .subsys_id      = NFNL_SUBSYS_NFTABLES,
10769         .cb_count       = NFT_MSG_MAX,
10770         .cb             = nf_tables_cb,
10771         .commit         = nf_tables_commit,
10772         .abort          = nf_tables_abort,
10773         .valid_genid    = nf_tables_valid_genid,
10774         .owner          = THIS_MODULE,
10775 };
10776
10777 int nft_chain_validate_dependency(const struct nft_chain *chain,
10778                                   enum nft_chain_types type)
10779 {
10780         const struct nft_base_chain *basechain;
10781
10782         if (nft_is_base_chain(chain)) {
10783                 basechain = nft_base_chain(chain);
10784                 if (basechain->type->type != type)
10785                         return -EOPNOTSUPP;
10786         }
10787         return 0;
10788 }
10789 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
10790
10791 int nft_chain_validate_hooks(const struct nft_chain *chain,
10792                              unsigned int hook_flags)
10793 {
10794         struct nft_base_chain *basechain;
10795
10796         if (nft_is_base_chain(chain)) {
10797                 basechain = nft_base_chain(chain);
10798
10799                 if ((1 << basechain->ops.hooknum) & hook_flags)
10800                         return 0;
10801
10802                 return -EOPNOTSUPP;
10803         }
10804
10805         return 0;
10806 }
10807 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
10808
10809 /*
10810  * Loop detection - walk through the ruleset beginning at the destination chain
10811  * of a new jump until either the source chain is reached (loop) or all
10812  * reachable chains have been traversed.
10813  *
10814  * The loop check is performed whenever a new jump verdict is added to an
10815  * expression or verdict map or a verdict map is bound to a new chain.
10816  */
10817
10818 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10819                                  const struct nft_chain *chain);
10820
10821 static int nft_check_loops(const struct nft_ctx *ctx,
10822                            const struct nft_set_ext *ext)
10823 {
10824         const struct nft_data *data;
10825         int ret;
10826
10827         data = nft_set_ext_data(ext);
10828         switch (data->verdict.code) {
10829         case NFT_JUMP:
10830         case NFT_GOTO:
10831                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
10832                 break;
10833         default:
10834                 ret = 0;
10835                 break;
10836         }
10837
10838         return ret;
10839 }
10840
10841 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
10842                                         struct nft_set *set,
10843                                         const struct nft_set_iter *iter,
10844                                         struct nft_elem_priv *elem_priv)
10845 {
10846         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
10847
10848         if (!nft_set_elem_active(ext, iter->genmask))
10849                 return 0;
10850
10851         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
10852             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
10853                 return 0;
10854
10855         return nft_check_loops(ctx, ext);
10856 }
10857
10858 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
10859                                   struct nft_set *set)
10860 {
10861         u8 genmask = nft_genmask_next(ctx->net);
10862         struct nft_set_elem_catchall *catchall;
10863         struct nft_set_ext *ext;
10864         int ret = 0;
10865
10866         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10867                 ext = nft_set_elem_ext(set, catchall->elem);
10868                 if (!nft_set_elem_active(ext, genmask))
10869                         continue;
10870
10871                 ret = nft_check_loops(ctx, ext);
10872                 if (ret < 0)
10873                         return ret;
10874         }
10875
10876         return ret;
10877 }
10878
10879 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10880                                  const struct nft_chain *chain)
10881 {
10882         const struct nft_rule *rule;
10883         const struct nft_expr *expr, *last;
10884         struct nft_set *set;
10885         struct nft_set_binding *binding;
10886         struct nft_set_iter iter;
10887
10888         if (ctx->chain == chain)
10889                 return -ELOOP;
10890
10891         if (fatal_signal_pending(current))
10892                 return -EINTR;
10893
10894         list_for_each_entry(rule, &chain->rules, list) {
10895                 nft_rule_for_each_expr(expr, last, rule) {
10896                         struct nft_immediate_expr *priv;
10897                         const struct nft_data *data;
10898                         int err;
10899
10900                         if (strcmp(expr->ops->type->name, "immediate"))
10901                                 continue;
10902
10903                         priv = nft_expr_priv(expr);
10904                         if (priv->dreg != NFT_REG_VERDICT)
10905                                 continue;
10906
10907                         data = &priv->data;
10908                         switch (data->verdict.code) {
10909                         case NFT_JUMP:
10910                         case NFT_GOTO:
10911                                 err = nf_tables_check_loops(ctx,
10912                                                         data->verdict.chain);
10913                                 if (err < 0)
10914                                         return err;
10915                                 break;
10916                         default:
10917                                 break;
10918                         }
10919                 }
10920         }
10921
10922         list_for_each_entry(set, &ctx->table->sets, list) {
10923                 if (!nft_is_active_next(ctx->net, set))
10924                         continue;
10925                 if (!(set->flags & NFT_SET_MAP) ||
10926                     set->dtype != NFT_DATA_VERDICT)
10927                         continue;
10928
10929                 list_for_each_entry(binding, &set->bindings, list) {
10930                         if (!(binding->flags & NFT_SET_MAP) ||
10931                             binding->chain != chain)
10932                                 continue;
10933
10934                         iter.genmask    = nft_genmask_next(ctx->net);
10935                         iter.type       = NFT_ITER_UPDATE;
10936                         iter.skip       = 0;
10937                         iter.count      = 0;
10938                         iter.err        = 0;
10939                         iter.fn         = nf_tables_loop_check_setelem;
10940
10941                         set->ops->walk(ctx, set, &iter);
10942                         if (!iter.err)
10943                                 iter.err = nft_set_catchall_loops(ctx, set);
10944
10945                         if (iter.err < 0)
10946                                 return iter.err;
10947                 }
10948         }
10949
10950         return 0;
10951 }
10952
10953 /**
10954  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10955  *
10956  *      @attr: netlink attribute to fetch value from
10957  *      @max: maximum value to be stored in dest
10958  *      @dest: pointer to the variable
10959  *
10960  *      Parse, check and store a given u32 netlink attribute into variable.
10961  *      This function returns -ERANGE if the value goes over maximum value.
10962  *      Otherwise a 0 is returned and the attribute value is stored in the
10963  *      destination variable.
10964  */
10965 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10966 {
10967         u32 val;
10968
10969         val = ntohl(nla_get_be32(attr));
10970         if (val > max)
10971                 return -ERANGE;
10972
10973         *dest = val;
10974         return 0;
10975 }
10976 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10977
10978 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10979 {
10980         unsigned int reg;
10981
10982         reg = ntohl(nla_get_be32(attr));
10983         switch (reg) {
10984         case NFT_REG_VERDICT...NFT_REG_4:
10985                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10986                 break;
10987         case NFT_REG32_00...NFT_REG32_15:
10988                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10989                 break;
10990         default:
10991                 return -ERANGE;
10992         }
10993
10994         return 0;
10995 }
10996
10997 /**
10998  *      nft_dump_register - dump a register value to a netlink attribute
10999  *
11000  *      @skb: socket buffer
11001  *      @attr: attribute number
11002  *      @reg: register number
11003  *
11004  *      Construct a netlink attribute containing the register number. For
11005  *      compatibility reasons, register numbers being a multiple of 4 are
11006  *      translated to the corresponding 128 bit register numbers.
11007  */
11008 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
11009 {
11010         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
11011                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
11012         else
11013                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
11014
11015         return nla_put_be32(skb, attr, htonl(reg));
11016 }
11017 EXPORT_SYMBOL_GPL(nft_dump_register);
11018
11019 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
11020 {
11021         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11022                 return -EINVAL;
11023         if (len == 0)
11024                 return -EINVAL;
11025         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
11026                 return -ERANGE;
11027
11028         return 0;
11029 }
11030
11031 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
11032 {
11033         u32 reg;
11034         int err;
11035
11036         err = nft_parse_register(attr, &reg);
11037         if (err < 0)
11038                 return err;
11039
11040         err = nft_validate_register_load(reg, len);
11041         if (err < 0)
11042                 return err;
11043
11044         *sreg = reg;
11045         return 0;
11046 }
11047 EXPORT_SYMBOL_GPL(nft_parse_register_load);
11048
11049 static int nft_validate_register_store(const struct nft_ctx *ctx,
11050                                        enum nft_registers reg,
11051                                        const struct nft_data *data,
11052                                        enum nft_data_types type,
11053                                        unsigned int len)
11054 {
11055         int err;
11056
11057         switch (reg) {
11058         case NFT_REG_VERDICT:
11059                 if (type != NFT_DATA_VERDICT)
11060                         return -EINVAL;
11061
11062                 if (data != NULL &&
11063                     (data->verdict.code == NFT_GOTO ||
11064                      data->verdict.code == NFT_JUMP)) {
11065                         err = nf_tables_check_loops(ctx, data->verdict.chain);
11066                         if (err < 0)
11067                                 return err;
11068                 }
11069
11070                 return 0;
11071         default:
11072                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11073                         return -EINVAL;
11074                 if (len == 0)
11075                         return -EINVAL;
11076                 if (reg * NFT_REG32_SIZE + len >
11077                     sizeof_field(struct nft_regs, data))
11078                         return -ERANGE;
11079
11080                 if (data != NULL && type != NFT_DATA_VALUE)
11081                         return -EINVAL;
11082                 return 0;
11083         }
11084 }
11085
11086 int nft_parse_register_store(const struct nft_ctx *ctx,
11087                              const struct nlattr *attr, u8 *dreg,
11088                              const struct nft_data *data,
11089                              enum nft_data_types type, unsigned int len)
11090 {
11091         int err;
11092         u32 reg;
11093
11094         err = nft_parse_register(attr, &reg);
11095         if (err < 0)
11096                 return err;
11097
11098         err = nft_validate_register_store(ctx, reg, data, type, len);
11099         if (err < 0)
11100                 return err;
11101
11102         *dreg = reg;
11103         return 0;
11104 }
11105 EXPORT_SYMBOL_GPL(nft_parse_register_store);
11106
11107 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
11108         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
11109         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
11110                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
11111         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
11112 };
11113
11114 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
11115                             struct nft_data_desc *desc, const struct nlattr *nla)
11116 {
11117         u8 genmask = nft_genmask_next(ctx->net);
11118         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
11119         struct nft_chain *chain;
11120         int err;
11121
11122         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
11123                                           nft_verdict_policy, NULL);
11124         if (err < 0)
11125                 return err;
11126
11127         if (!tb[NFTA_VERDICT_CODE])
11128                 return -EINVAL;
11129
11130         /* zero padding hole for memcmp */
11131         memset(data, 0, sizeof(*data));
11132         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
11133
11134         switch (data->verdict.code) {
11135         case NF_ACCEPT:
11136         case NF_DROP:
11137         case NF_QUEUE:
11138                 break;
11139         case NFT_CONTINUE:
11140         case NFT_BREAK:
11141         case NFT_RETURN:
11142                 break;
11143         case NFT_JUMP:
11144         case NFT_GOTO:
11145                 if (tb[NFTA_VERDICT_CHAIN]) {
11146                         chain = nft_chain_lookup(ctx->net, ctx->table,
11147                                                  tb[NFTA_VERDICT_CHAIN],
11148                                                  genmask);
11149                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
11150                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
11151                                                       tb[NFTA_VERDICT_CHAIN_ID],
11152                                                       genmask);
11153                         if (IS_ERR(chain))
11154                                 return PTR_ERR(chain);
11155                 } else {
11156                         return -EINVAL;
11157                 }
11158
11159                 if (IS_ERR(chain))
11160                         return PTR_ERR(chain);
11161                 if (nft_is_base_chain(chain))
11162                         return -EOPNOTSUPP;
11163                 if (nft_chain_is_bound(chain))
11164                         return -EINVAL;
11165                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
11166                     chain->flags & NFT_CHAIN_BINDING)
11167                         return -EINVAL;
11168                 if (!nft_use_inc(&chain->use))
11169                         return -EMFILE;
11170
11171                 data->verdict.chain = chain;
11172                 break;
11173         default:
11174                 return -EINVAL;
11175         }
11176
11177         desc->len = sizeof(data->verdict);
11178
11179         return 0;
11180 }
11181
11182 static void nft_verdict_uninit(const struct nft_data *data)
11183 {
11184         struct nft_chain *chain;
11185
11186         switch (data->verdict.code) {
11187         case NFT_JUMP:
11188         case NFT_GOTO:
11189                 chain = data->verdict.chain;
11190                 nft_use_dec(&chain->use);
11191                 break;
11192         }
11193 }
11194
11195 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
11196 {
11197         struct nlattr *nest;
11198
11199         nest = nla_nest_start_noflag(skb, type);
11200         if (!nest)
11201                 goto nla_put_failure;
11202
11203         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
11204                 goto nla_put_failure;
11205
11206         switch (v->code) {
11207         case NFT_JUMP:
11208         case NFT_GOTO:
11209                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
11210                                    v->chain->name))
11211                         goto nla_put_failure;
11212         }
11213         nla_nest_end(skb, nest);
11214         return 0;
11215
11216 nla_put_failure:
11217         return -1;
11218 }
11219
11220 static int nft_value_init(const struct nft_ctx *ctx,
11221                           struct nft_data *data, struct nft_data_desc *desc,
11222                           const struct nlattr *nla)
11223 {
11224         unsigned int len;
11225
11226         len = nla_len(nla);
11227         if (len == 0)
11228                 return -EINVAL;
11229         if (len > desc->size)
11230                 return -EOVERFLOW;
11231         if (desc->len) {
11232                 if (len != desc->len)
11233                         return -EINVAL;
11234         } else {
11235                 desc->len = len;
11236         }
11237
11238         nla_memcpy(data->data, nla, len);
11239
11240         return 0;
11241 }
11242
11243 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
11244                           unsigned int len)
11245 {
11246         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
11247 }
11248
11249 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
11250         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
11251         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
11252 };
11253
11254 /**
11255  *      nft_data_init - parse nf_tables data netlink attributes
11256  *
11257  *      @ctx: context of the expression using the data
11258  *      @data: destination struct nft_data
11259  *      @desc: data description
11260  *      @nla: netlink attribute containing data
11261  *
11262  *      Parse the netlink data attributes and initialize a struct nft_data.
11263  *      The type and length of data are returned in the data description.
11264  *
11265  *      The caller can indicate that it only wants to accept data of type
11266  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
11267  */
11268 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
11269                   struct nft_data_desc *desc, const struct nlattr *nla)
11270 {
11271         struct nlattr *tb[NFTA_DATA_MAX + 1];
11272         int err;
11273
11274         if (WARN_ON_ONCE(!desc->size))
11275                 return -EINVAL;
11276
11277         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
11278                                           nft_data_policy, NULL);
11279         if (err < 0)
11280                 return err;
11281
11282         if (tb[NFTA_DATA_VALUE]) {
11283                 if (desc->type != NFT_DATA_VALUE)
11284                         return -EINVAL;
11285
11286                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
11287         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
11288                 if (desc->type != NFT_DATA_VERDICT)
11289                         return -EINVAL;
11290
11291                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
11292         } else {
11293                 err = -EINVAL;
11294         }
11295
11296         return err;
11297 }
11298 EXPORT_SYMBOL_GPL(nft_data_init);
11299
11300 /**
11301  *      nft_data_release - release a nft_data item
11302  *
11303  *      @data: struct nft_data to release
11304  *      @type: type of data
11305  *
11306  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
11307  *      all others need to be released by calling this function.
11308  */
11309 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
11310 {
11311         if (type < NFT_DATA_VERDICT)
11312                 return;
11313         switch (type) {
11314         case NFT_DATA_VERDICT:
11315                 return nft_verdict_uninit(data);
11316         default:
11317                 WARN_ON(1);
11318         }
11319 }
11320 EXPORT_SYMBOL_GPL(nft_data_release);
11321
11322 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
11323                   enum nft_data_types type, unsigned int len)
11324 {
11325         struct nlattr *nest;
11326         int err;
11327
11328         nest = nla_nest_start_noflag(skb, attr);
11329         if (nest == NULL)
11330                 return -1;
11331
11332         switch (type) {
11333         case NFT_DATA_VALUE:
11334                 err = nft_value_dump(skb, data, len);
11335                 break;
11336         case NFT_DATA_VERDICT:
11337                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
11338                 break;
11339         default:
11340                 err = -EINVAL;
11341                 WARN_ON(1);
11342         }
11343
11344         nla_nest_end(skb, nest);
11345         return err;
11346 }
11347 EXPORT_SYMBOL_GPL(nft_data_dump);
11348
11349 int __nft_release_basechain(struct nft_ctx *ctx)
11350 {
11351         struct nft_rule *rule, *nr;
11352
11353         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
11354                 return 0;
11355
11356         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
11357         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
11358                 list_del(&rule->list);
11359                 nft_use_dec(&ctx->chain->use);
11360                 nf_tables_rule_release(ctx, rule);
11361         }
11362         nft_chain_del(ctx->chain);
11363         nft_use_dec(&ctx->table->use);
11364         nf_tables_chain_destroy(ctx);
11365
11366         return 0;
11367 }
11368 EXPORT_SYMBOL_GPL(__nft_release_basechain);
11369
11370 static void __nft_release_hook(struct net *net, struct nft_table *table)
11371 {
11372         struct nft_flowtable *flowtable;
11373         struct nft_chain *chain;
11374
11375         list_for_each_entry(chain, &table->chains, list)
11376                 __nf_tables_unregister_hook(net, table, chain, true);
11377         list_for_each_entry(flowtable, &table->flowtables, list)
11378                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
11379                                                      true);
11380 }
11381
11382 static void __nft_release_hooks(struct net *net)
11383 {
11384         struct nftables_pernet *nft_net = nft_pernet(net);
11385         struct nft_table *table;
11386
11387         list_for_each_entry(table, &nft_net->tables, list) {
11388                 if (nft_table_has_owner(table))
11389                         continue;
11390
11391                 __nft_release_hook(net, table);
11392         }
11393 }
11394
11395 static void __nft_release_table(struct net *net, struct nft_table *table)
11396 {
11397         struct nft_flowtable *flowtable, *nf;
11398         struct nft_chain *chain, *nc;
11399         struct nft_object *obj, *ne;
11400         struct nft_rule *rule, *nr;
11401         struct nft_set *set, *ns;
11402         struct nft_ctx ctx = {
11403                 .net    = net,
11404                 .family = NFPROTO_NETDEV,
11405         };
11406
11407         ctx.family = table->family;
11408         ctx.table = table;
11409         list_for_each_entry(chain, &table->chains, list) {
11410                 if (nft_chain_binding(chain))
11411                         continue;
11412
11413                 ctx.chain = chain;
11414                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
11415                         list_del(&rule->list);
11416                         nft_use_dec(&chain->use);
11417                         nf_tables_rule_release(&ctx, rule);
11418                 }
11419         }
11420         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
11421                 list_del(&flowtable->list);
11422                 nft_use_dec(&table->use);
11423                 nf_tables_flowtable_destroy(flowtable);
11424         }
11425         list_for_each_entry_safe(set, ns, &table->sets, list) {
11426                 list_del(&set->list);
11427                 nft_use_dec(&table->use);
11428                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11429                         nft_map_deactivate(&ctx, set);
11430
11431                 nft_set_destroy(&ctx, set);
11432         }
11433         list_for_each_entry_safe(obj, ne, &table->objects, list) {
11434                 nft_obj_del(obj);
11435                 nft_use_dec(&table->use);
11436                 nft_obj_destroy(&ctx, obj);
11437         }
11438         list_for_each_entry_safe(chain, nc, &table->chains, list) {
11439                 ctx.chain = chain;
11440                 nft_chain_del(chain);
11441                 nft_use_dec(&table->use);
11442                 nf_tables_chain_destroy(&ctx);
11443         }
11444         nf_tables_table_destroy(&ctx);
11445 }
11446
11447 static void __nft_release_tables(struct net *net)
11448 {
11449         struct nftables_pernet *nft_net = nft_pernet(net);
11450         struct nft_table *table, *nt;
11451
11452         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
11453                 if (nft_table_has_owner(table))
11454                         continue;
11455
11456                 list_del(&table->list);
11457
11458                 __nft_release_table(net, table);
11459         }
11460 }
11461
11462 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
11463                             void *ptr)
11464 {
11465         struct nft_table *table, *to_delete[8];
11466         struct nftables_pernet *nft_net;
11467         struct netlink_notify *n = ptr;
11468         struct net *net = n->net;
11469         unsigned int deleted;
11470         bool restart = false;
11471         unsigned int gc_seq;
11472
11473         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
11474                 return NOTIFY_DONE;
11475
11476         nft_net = nft_pernet(net);
11477         deleted = 0;
11478         mutex_lock(&nft_net->commit_mutex);
11479
11480         gc_seq = nft_gc_seq_begin(nft_net);
11481
11482         if (!list_empty(&nf_tables_destroy_list))
11483                 nf_tables_trans_destroy_flush_work();
11484 again:
11485         list_for_each_entry(table, &nft_net->tables, list) {
11486                 if (nft_table_has_owner(table) &&
11487                     n->portid == table->nlpid) {
11488                         if (table->flags & NFT_TABLE_F_PERSIST) {
11489                                 table->flags &= ~NFT_TABLE_F_OWNER;
11490                                 continue;
11491                         }
11492                         __nft_release_hook(net, table);
11493                         list_del_rcu(&table->list);
11494                         to_delete[deleted++] = table;
11495                         if (deleted >= ARRAY_SIZE(to_delete))
11496                                 break;
11497                 }
11498         }
11499         if (deleted) {
11500                 restart = deleted >= ARRAY_SIZE(to_delete);
11501                 synchronize_rcu();
11502                 while (deleted)
11503                         __nft_release_table(net, to_delete[--deleted]);
11504
11505                 if (restart)
11506                         goto again;
11507         }
11508         nft_gc_seq_end(nft_net, gc_seq);
11509
11510         mutex_unlock(&nft_net->commit_mutex);
11511
11512         return NOTIFY_DONE;
11513 }
11514
11515 static struct notifier_block nft_nl_notifier = {
11516         .notifier_call  = nft_rcv_nl_event,
11517 };
11518
11519 static int __net_init nf_tables_init_net(struct net *net)
11520 {
11521         struct nftables_pernet *nft_net = nft_pernet(net);
11522
11523         INIT_LIST_HEAD(&nft_net->tables);
11524         INIT_LIST_HEAD(&nft_net->commit_list);
11525         INIT_LIST_HEAD(&nft_net->binding_list);
11526         INIT_LIST_HEAD(&nft_net->module_list);
11527         INIT_LIST_HEAD(&nft_net->notify_list);
11528         mutex_init(&nft_net->commit_mutex);
11529         nft_net->base_seq = 1;
11530         nft_net->gc_seq = 0;
11531         nft_net->validate_state = NFT_VALIDATE_SKIP;
11532
11533         return 0;
11534 }
11535
11536 static void __net_exit nf_tables_pre_exit_net(struct net *net)
11537 {
11538         struct nftables_pernet *nft_net = nft_pernet(net);
11539
11540         mutex_lock(&nft_net->commit_mutex);
11541         __nft_release_hooks(net);
11542         mutex_unlock(&nft_net->commit_mutex);
11543 }
11544
11545 static void __net_exit nf_tables_exit_net(struct net *net)
11546 {
11547         struct nftables_pernet *nft_net = nft_pernet(net);
11548         unsigned int gc_seq;
11549
11550         mutex_lock(&nft_net->commit_mutex);
11551
11552         gc_seq = nft_gc_seq_begin(nft_net);
11553
11554         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
11555
11556         if (!list_empty(&nft_net->module_list))
11557                 nf_tables_module_autoload_cleanup(net);
11558
11559         __nft_release_tables(net);
11560
11561         nft_gc_seq_end(nft_net, gc_seq);
11562
11563         mutex_unlock(&nft_net->commit_mutex);
11564         WARN_ON_ONCE(!list_empty(&nft_net->tables));
11565         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
11566         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
11567 }
11568
11569 static void nf_tables_exit_batch(struct list_head *net_exit_list)
11570 {
11571         flush_work(&trans_gc_work);
11572 }
11573
11574 static struct pernet_operations nf_tables_net_ops = {
11575         .init           = nf_tables_init_net,
11576         .pre_exit       = nf_tables_pre_exit_net,
11577         .exit           = nf_tables_exit_net,
11578         .exit_batch     = nf_tables_exit_batch,
11579         .id             = &nf_tables_net_id,
11580         .size           = sizeof(struct nftables_pernet),
11581 };
11582
11583 static int __init nf_tables_module_init(void)
11584 {
11585         int err;
11586
11587         err = register_pernet_subsys(&nf_tables_net_ops);
11588         if (err < 0)
11589                 return err;
11590
11591         err = nft_chain_filter_init();
11592         if (err < 0)
11593                 goto err_chain_filter;
11594
11595         err = nf_tables_core_module_init();
11596         if (err < 0)
11597                 goto err_core_module;
11598
11599         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
11600         if (err < 0)
11601                 goto err_netdev_notifier;
11602
11603         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
11604         if (err < 0)
11605                 goto err_rht_objname;
11606
11607         err = nft_offload_init();
11608         if (err < 0)
11609                 goto err_offload;
11610
11611         err = netlink_register_notifier(&nft_nl_notifier);
11612         if (err < 0)
11613                 goto err_netlink_notifier;
11614
11615         /* must be last */
11616         err = nfnetlink_subsys_register(&nf_tables_subsys);
11617         if (err < 0)
11618                 goto err_nfnl_subsys;
11619
11620         nft_chain_route_init();
11621
11622         return err;
11623
11624 err_nfnl_subsys:
11625         netlink_unregister_notifier(&nft_nl_notifier);
11626 err_netlink_notifier:
11627         nft_offload_exit();
11628 err_offload:
11629         rhltable_destroy(&nft_objname_ht);
11630 err_rht_objname:
11631         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11632 err_netdev_notifier:
11633         nf_tables_core_module_exit();
11634 err_core_module:
11635         nft_chain_filter_fini();
11636 err_chain_filter:
11637         unregister_pernet_subsys(&nf_tables_net_ops);
11638         return err;
11639 }
11640
11641 static void __exit nf_tables_module_exit(void)
11642 {
11643         nfnetlink_subsys_unregister(&nf_tables_subsys);
11644         netlink_unregister_notifier(&nft_nl_notifier);
11645         nft_offload_exit();
11646         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11647         nft_chain_filter_fini();
11648         nft_chain_route_fini();
11649         nf_tables_trans_destroy_flush_work();
11650         unregister_pernet_subsys(&nf_tables_net_ops);
11651         cancel_work_sync(&trans_gc_work);
11652         cancel_work_sync(&trans_destroy_work);
11653         rcu_barrier();
11654         rhltable_destroy(&nft_objname_ht);
11655         nf_tables_core_module_exit();
11656 }
11657
11658 module_init(nf_tables_module_init);
11659 module_exit(nf_tables_module_exit);
11660
11661 MODULE_LICENSE("GPL");
11662 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
11663 MODULE_DESCRIPTION("Framework for packet filtering and classification");
11664 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);