net/sched/cls_api.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * net/sched/cls_api.c  Packet classifier API.
   4  *
   5  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
   6  *
   7  * Changes:
   8  *
   9  * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
  10  */
  11
  12 #include <linux/module.h>
  13 #include <linux/types.h>
  14 #include <linux/kernel.h>
  15 #include <linux/string.h>
  16 #include <linux/errno.h>
  17 #include <linux/err.h>
  18 #include <linux/skbuff.h>
  19 #include <linux/init.h>
  20 #include <linux/kmod.h>
  21 #include <linux/slab.h>
  22 #include <linux/idr.h>
  23 #include <linux/rhashtable.h>
  24 #include <linux/jhash.h>
  25 #include <net/net_namespace.h>
  26 #include <net/sock.h>
  27 #include <net/netlink.h>
  28 #include <net/pkt_sched.h>
  29 #include <net/pkt_cls.h>
  30 #include <net/tc_act/tc_pedit.h>
  31 #include <net/tc_act/tc_mirred.h>
  32 #include <net/tc_act/tc_vlan.h>
  33 #include <net/tc_act/tc_tunnel_key.h>
  34 #include <net/tc_act/tc_csum.h>
  35 #include <net/tc_act/tc_gact.h>
  36 #include <net/tc_act/tc_police.h>
  37 #include <net/tc_act/tc_sample.h>
  38 #include <net/tc_act/tc_skbedit.h>
  39 #include <net/tc_act/tc_ct.h>
  40 #include <net/tc_act/tc_mpls.h>
  41 #include <net/flow_offload.h>
  42
  43 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
  44
  45 /* The list of all installed classifier types */
  46 static LIST_HEAD(tcf_proto_base);
  47
  48 /* Protects list of registered TC modules. It is pure SMP lock. */
  49 static DEFINE_RWLOCK(cls_mod_lock);
  50
  51 static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
  52 {
  53         return jhash_3words(tp->chain->index, tp->prio,
  54                             (__force __u32)tp->protocol, 0);
  55 }
  56
  57 static void tcf_proto_signal_destroying(struct tcf_chain *chain,
  58                                         struct tcf_proto *tp)
  59 {
  60         struct tcf_block *block = chain->block;
  61
  62         mutex_lock(&block->proto_destroy_lock);
  63         hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
  64                      destroy_obj_hashfn(tp));
  65         mutex_unlock(&block->proto_destroy_lock);
  66 }
  67
  68 static bool tcf_proto_cmp(const struct tcf_proto *tp1,
  69                           const struct tcf_proto *tp2)
  70 {
  71         return tp1->chain->index == tp2->chain->index &&
  72                tp1->prio == tp2->prio &&
  73                tp1->protocol == tp2->protocol;
  74 }
  75
  76 static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
  77                                         struct tcf_proto *tp)
  78 {
  79         u32 hash = destroy_obj_hashfn(tp);
  80         struct tcf_proto *iter;
  81         bool found = false;
  82
  83         rcu_read_lock();
  84         hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
  85                                    destroy_ht_node, hash) {
  86                 if (tcf_proto_cmp(tp, iter)) {
  87                         found = true;
  88                         break;
  89                 }
  90         }
  91         rcu_read_unlock();
  92
  93         return found;
  94 }
  95
  96 static void
  97 tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
  98 {
  99         struct tcf_block *block = chain->block;
 100
 101         mutex_lock(&block->proto_destroy_lock);
 102         if (hash_hashed(&tp->destroy_ht_node))
 103                 hash_del_rcu(&tp->destroy_ht_node);
 104         mutex_unlock(&block->proto_destroy_lock);
 105 }
 106
 107 /* Find classifier type by string name */
 108
 109 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
 110 {
 111         const struct tcf_proto_ops *t, *res = NULL;
 112
 113         if (kind) {
 114                 read_lock(&cls_mod_lock);
 115                 list_for_each_entry(t, &tcf_proto_base, head) {
 116                         if (strcmp(kind, t->kind) == 0) {
 117                                 if (try_module_get(t->owner))
 118                                         res = t;
 119                                 break;
 120                         }
 121                 }
 122                 read_unlock(&cls_mod_lock);
 123         }
 124         return res;
 125 }
 126
 127 static const struct tcf_proto_ops *
 128 tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
 129                      struct netlink_ext_ack *extack)
 130 {
 131         const struct tcf_proto_ops *ops;
 132
 133         ops = __tcf_proto_lookup_ops(kind);
 134         if (ops)
 135                 return ops;
 136 #ifdef CONFIG_MODULES
 137         if (rtnl_held)
 138                 rtnl_unlock();
 139         request_module("cls_%s", kind);
 140         if (rtnl_held)
 141                 rtnl_lock();
 142         ops = __tcf_proto_lookup_ops(kind);
 143         /* We dropped the RTNL semaphore in order to perform
 144          * the module load. So, even if we succeeded in loading
 145          * the module we have to replay the request. We indicate
 146          * this using -EAGAIN.
 147          */
 148         if (ops) {
 149                 module_put(ops->owner);
 150                 return ERR_PTR(-EAGAIN);
 151         }
 152 #endif
 153         NL_SET_ERR_MSG(extack, "TC classifier not found");
 154         return ERR_PTR(-ENOENT);
 155 }
 156
 157 /* Register(unregister) new classifier type */
 158
 159 int register_tcf_proto_ops(struct tcf_proto_ops *ops)
 160 {
 161         struct tcf_proto_ops *t;
 162         int rc = -EEXIST;
 163
 164         write_lock(&cls_mod_lock);
 165         list_for_each_entry(t, &tcf_proto_base, head)
 166                 if (!strcmp(ops->kind, t->kind))
 167                         goto out;
 168
 169         list_add_tail(&ops->head, &tcf_proto_base);
 170         rc = 0;
 171 out:
 172         write_unlock(&cls_mod_lock);
 173         return rc;
 174 }
 175 EXPORT_SYMBOL(register_tcf_proto_ops);
 176
 177 static struct workqueue_struct *tc_filter_wq;
 178
 179 int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
 180 {
 181         struct tcf_proto_ops *t;
 182         int rc = -ENOENT;
 183
 184         /* Wait for outstanding call_rcu()s, if any, from a
 185          * tcf_proto_ops's destroy() handler.
 186          */
 187         rcu_barrier();
 188         flush_workqueue(tc_filter_wq);
 189
 190         write_lock(&cls_mod_lock);
 191         list_for_each_entry(t, &tcf_proto_base, head) {
 192                 if (t == ops) {
 193                         list_del(&t->head);
 194                         rc = 0;
 195                         break;
 196                 }
 197         }
 198         write_unlock(&cls_mod_lock);
 199         return rc;
 200 }
 201 EXPORT_SYMBOL(unregister_tcf_proto_ops);
 202
 203 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
 204 {
 205         INIT_RCU_WORK(rwork, func);
 206         return queue_rcu_work(tc_filter_wq, rwork);
 207 }
 208 EXPORT_SYMBOL(tcf_queue_work);
 209
 210 /* Select new prio value from the range, managed by kernel. */
 211
 212 static inline u32 tcf_auto_prio(struct tcf_proto *tp)
 213 {
 214         u32 first = TC_H_MAKE(0xC0000000U, 0U);
 215
 216         if (tp)
 217                 first = tp->prio - 1;
 218
 219         return TC_H_MAJ(first);
 220 }
 221
 222 static bool tcf_proto_check_kind(struct nlattr *kind, char *name)
 223 {
 224         if (kind)
 225                 return nla_strlcpy(name, kind, IFNAMSIZ) >= IFNAMSIZ;
 226         memset(name, 0, IFNAMSIZ);
 227         return false;
 228 }
 229
 230 static bool tcf_proto_is_unlocked(const char *kind)
 231 {
 232         const struct tcf_proto_ops *ops;
 233         bool ret;
 234
 235         if (strlen(kind) == 0)
 236                 return false;
 237
 238         ops = tcf_proto_lookup_ops(kind, false, NULL);
 239         /* On error return false to take rtnl lock. Proto lookup/create
 240          * functions will perform lookup again and properly handle errors.
 241          */
 242         if (IS_ERR(ops))
 243                 return false;
 244
 245         ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
 246         module_put(ops->owner);
 247         return ret;
 248 }
 249
 250 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
 251                                           u32 prio, struct tcf_chain *chain,
 252                                           bool rtnl_held,
 253                                           struct netlink_ext_ack *extack)
 254 {
 255         struct tcf_proto *tp;
 256         int err;
 257
 258         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 259         if (!tp)
 260                 return ERR_PTR(-ENOBUFS);
 261
 262         tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
 263         if (IS_ERR(tp->ops)) {
 264                 err = PTR_ERR(tp->ops);
 265                 goto errout;
 266         }
 267         tp->classify = tp->ops->classify;
 268         tp->protocol = protocol;
 269         tp->prio = prio;
 270         tp->chain = chain;
 271         spin_lock_init(&tp->lock);
 272         refcount_set(&tp->refcnt, 1);
 273
 274         err = tp->ops->init(tp);
 275         if (err) {
 276                 module_put(tp->ops->owner);
 277                 goto errout;
 278         }
 279         return tp;
 280
 281 errout:
 282         kfree(tp);
 283         return ERR_PTR(err);
 284 }
 285
 286 static void tcf_proto_get(struct tcf_proto *tp)
 287 {
 288         refcount_inc(&tp->refcnt);
 289 }
 290
 291 static void tcf_chain_put(struct tcf_chain *chain);
 292
 293 static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
 294                               bool sig_destroy, struct netlink_ext_ack *extack)
 295 {
 296         tp->ops->destroy(tp, rtnl_held, extack);
 297         if (sig_destroy)
 298                 tcf_proto_signal_destroyed(tp->chain, tp);
 299         tcf_chain_put(tp->chain);
 300         module_put(tp->ops->owner);
 301         kfree_rcu(tp, rcu);
 302 }
 303
 304 static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
 305                           struct netlink_ext_ack *extack)
 306 {
 307         if (refcount_dec_and_test(&tp->refcnt))
 308                 tcf_proto_destroy(tp, rtnl_held, true, extack);
 309 }
 310
 311 static bool tcf_proto_check_delete(struct tcf_proto *tp)
 312 {
 313         if (tp->ops->delete_empty)
 314                 return tp->ops->delete_empty(tp);
 315
 316         tp->deleting = true;
 317         return tp->deleting;
 318 }
 319
 320 static void tcf_proto_mark_delete(struct tcf_proto *tp)
 321 {
 322         spin_lock(&tp->lock);
 323         tp->deleting = true;
 324         spin_unlock(&tp->lock);
 325 }
 326
 327 static bool tcf_proto_is_deleting(struct tcf_proto *tp)
 328 {
 329         bool deleting;
 330
 331         spin_lock(&tp->lock);
 332         deleting = tp->deleting;
 333         spin_unlock(&tp->lock);
 334
 335         return deleting;
 336 }
 337
 338 #define ASSERT_BLOCK_LOCKED(block)                                      \
 339         lockdep_assert_held(&(block)->lock)
 340
 341 struct tcf_filter_chain_list_item {
 342         struct list_head list;
 343         tcf_chain_head_change_t *chain_head_change;
 344         void *chain_head_change_priv;
 345 };
 346
 347 static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
 348                                           u32 chain_index)
 349 {
 350         struct tcf_chain *chain;
 351
 352         ASSERT_BLOCK_LOCKED(block);
 353
 354         chain = kzalloc(sizeof(*chain), GFP_KERNEL);
 355         if (!chain)
 356                 return NULL;
 357         list_add_tail(&chain->list, &block->chain_list);
 358         mutex_init(&chain->filter_chain_lock);
 359         chain->block = block;
 360         chain->index = chain_index;
 361         chain->refcnt = 1;
 362         if (!chain->index)
 363                 block->chain0.chain = chain;
 364         return chain;
 365 }
 366
 367 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
 368                                        struct tcf_proto *tp_head)
 369 {
 370         if (item->chain_head_change)
 371                 item->chain_head_change(tp_head, item->chain_head_change_priv);
 372 }
 373
 374 static void tcf_chain0_head_change(struct tcf_chain *chain,
 375                                    struct tcf_proto *tp_head)
 376 {
 377         struct tcf_filter_chain_list_item *item;
 378         struct tcf_block *block = chain->block;
 379
 380         if (chain->index)
 381                 return;
 382
 383         mutex_lock(&block->lock);
 384         list_for_each_entry(item, &block->chain0.filter_chain_list, list)
 385                 tcf_chain_head_change_item(item, tp_head);
 386         mutex_unlock(&block->lock);
 387 }
 388
 389 /* Returns true if block can be safely freed. */
 390
 391 static bool tcf_chain_detach(struct tcf_chain *chain)
 392 {
 393         struct tcf_block *block = chain->block;
 394
 395         ASSERT_BLOCK_LOCKED(block);
 396
 397         list_del(&chain->list);
 398         if (!chain->index)
 399                 block->chain0.chain = NULL;
 400
 401         if (list_empty(&block->chain_list) &&
 402             refcount_read(&block->refcnt) == 0)
 403                 return true;
 404
 405         return false;
 406 }
 407
 408 static void tcf_block_destroy(struct tcf_block *block)
 409 {
 410         mutex_destroy(&block->lock);
 411         mutex_destroy(&block->proto_destroy_lock);
 412         kfree_rcu(block, rcu);
 413 }
 414
 415 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
 416 {
 417         struct tcf_block *block = chain->block;
 418
 419         mutex_destroy(&chain->filter_chain_lock);
 420         kfree_rcu(chain, rcu);
 421         if (free_block)
 422                 tcf_block_destroy(block);
 423 }
 424
 425 static void tcf_chain_hold(struct tcf_chain *chain)
 426 {
 427         ASSERT_BLOCK_LOCKED(chain->block);
 428
 429         ++chain->refcnt;
 430 }
 431
 432 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
 433 {
 434         ASSERT_BLOCK_LOCKED(chain->block);
 435
 436         /* In case all the references are action references, this
 437          * chain should not be shown to the user.
 438          */
 439         return chain->refcnt == chain->action_refcnt;
 440 }
 441
 442 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
 443                                           u32 chain_index)
 444 {
 445         struct tcf_chain *chain;
 446
 447         ASSERT_BLOCK_LOCKED(block);
 448
 449         list_for_each_entry(chain, &block->chain_list, list) {
 450                 if (chain->index == chain_index)
 451                         return chain;
 452         }
 453         return NULL;
 454 }
 455
 456 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
 457                            u32 seq, u16 flags, int event, bool unicast);
 458
 459 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
 460                                          u32 chain_index, bool create,
 461                                          bool by_act)
 462 {
 463         struct tcf_chain *chain = NULL;
 464         bool is_first_reference;
 465
 466         mutex_lock(&block->lock);
 467         chain = tcf_chain_lookup(block, chain_index);
 468         if (chain) {
 469                 tcf_chain_hold(chain);
 470         } else {
 471                 if (!create)
 472                         goto errout;
 473                 chain = tcf_chain_create(block, chain_index);
 474                 if (!chain)
 475                         goto errout;
 476         }
 477
 478         if (by_act)
 479                 ++chain->action_refcnt;
 480         is_first_reference = chain->refcnt - chain->action_refcnt == 1;
 481         mutex_unlock(&block->lock);
 482
 483         /* Send notification only in case we got the first
 484          * non-action reference. Until then, the chain acts only as
 485          * a placeholder for actions pointing to it and user ought
 486          * not know about them.
 487          */
 488         if (is_first_reference && !by_act)
 489                 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
 490                                 RTM_NEWCHAIN, false);
 491
 492         return chain;
 493
 494 errout:
 495         mutex_unlock(&block->lock);
 496         return chain;
 497 }
 498
 499 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
 500                                        bool create)
 501 {
 502         return __tcf_chain_get(block, chain_index, create, false);
 503 }
 504
 505 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
 506 {
 507         return __tcf_chain_get(block, chain_index, true, true);
 508 }
 509 EXPORT_SYMBOL(tcf_chain_get_by_act);
 510
 511 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
 512                                void *tmplt_priv);
 513 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
 514                                   void *tmplt_priv, u32 chain_index,
 515                                   struct tcf_block *block, struct sk_buff *oskb,
 516                                   u32 seq, u16 flags, bool unicast);
 517
 518 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
 519                             bool explicitly_created)
 520 {
 521         struct tcf_block *block = chain->block;
 522         const struct tcf_proto_ops *tmplt_ops;
 523         bool free_block = false;
 524         unsigned int refcnt;
 525         void *tmplt_priv;
 526
 527         mutex_lock(&block->lock);
 528         if (explicitly_created) {
 529                 if (!chain->explicitly_created) {
 530                         mutex_unlock(&block->lock);
 531                         return;
 532                 }
 533                 chain->explicitly_created = false;
 534         }
 535
 536         if (by_act)
 537                 chain->action_refcnt--;
 538
 539         /* tc_chain_notify_delete can't be called while holding block lock.
 540          * However, when block is unlocked chain can be changed concurrently, so
 541          * save these to temporary variables.
 542          */
 543         refcnt = --chain->refcnt;
 544         tmplt_ops = chain->tmplt_ops;
 545         tmplt_priv = chain->tmplt_priv;
 546
 547         /* The last dropped non-action reference will trigger notification. */
 548         if (refcnt - chain->action_refcnt == 0 && !by_act) {
 549                 tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index,
 550                                        block, NULL, 0, 0, false);
 551                 /* Last reference to chain, no need to lock. */
 552                 chain->flushing = false;
 553         }
 554
 555         if (refcnt == 0)
 556                 free_block = tcf_chain_detach(chain);
 557         mutex_unlock(&block->lock);
 558
 559         if (refcnt == 0) {
 560                 tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
 561                 tcf_chain_destroy(chain, free_block);
 562         }
 563 }
 564
 565 static void tcf_chain_put(struct tcf_chain *chain)
 566 {
 567         __tcf_chain_put(chain, false, false);
 568 }
 569
 570 void tcf_chain_put_by_act(struct tcf_chain *chain)
 571 {
 572         __tcf_chain_put(chain, true, false);
 573 }
 574 EXPORT_SYMBOL(tcf_chain_put_by_act);
 575
 576 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
 577 {
 578         __tcf_chain_put(chain, false, true);
 579 }
 580
 581 static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
 582 {
 583         struct tcf_proto *tp, *tp_next;
 584
 585         mutex_lock(&chain->filter_chain_lock);
 586         tp = tcf_chain_dereference(chain->filter_chain, chain);
 587         while (tp) {
 588                 tp_next = rcu_dereference_protected(tp->next, 1);
 589                 tcf_proto_signal_destroying(chain, tp);
 590                 tp = tp_next;
 591         }
 592         tp = tcf_chain_dereference(chain->filter_chain, chain);
 593         RCU_INIT_POINTER(chain->filter_chain, NULL);
 594         tcf_chain0_head_change(chain, NULL);
 595         chain->flushing = true;
 596         mutex_unlock(&chain->filter_chain_lock);
 597
 598         while (tp) {
 599                 tp_next = rcu_dereference_protected(tp->next, 1);
 600                 tcf_proto_put(tp, rtnl_held, NULL);
 601                 tp = tp_next;
 602         }
 603 }
 604
 605 static int tcf_block_setup(struct tcf_block *block,
 606                            struct flow_block_offload *bo);
 607
 608 static void tc_indr_block_cmd(struct net_device *dev, struct tcf_block *block,
 609                               flow_indr_block_bind_cb_t *cb, void *cb_priv,
 610                               enum flow_block_command command, bool ingress)
 611 {
 612         struct flow_block_offload bo = {
 613                 .command        = command,
 614                 .binder_type    = ingress ?
 615                                   FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS :
 616                                   FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS,
 617                 .net            = dev_net(dev),
 618                 .block_shared   = tcf_block_non_null_shared(block),
 619         };
 620         INIT_LIST_HEAD(&bo.cb_list);
 621
 622         if (!block)
 623                 return;
 624
 625         bo.block = &block->flow_block;
 626
 627         down_write(&block->cb_lock);
 628         cb(dev, cb_priv, TC_SETUP_BLOCK, &bo);
 629
 630         tcf_block_setup(block, &bo);
 631         up_write(&block->cb_lock);
 632 }
 633
 634 static struct tcf_block *tc_dev_block(struct net_device *dev, bool ingress)
 635 {
 636         const struct Qdisc_class_ops *cops;
 637         const struct Qdisc_ops *ops;
 638         struct Qdisc *qdisc;
 639
 640         if (!dev_ingress_queue(dev))
 641                 return NULL;
 642
 643         qdisc = dev_ingress_queue(dev)->qdisc_sleeping;
 644         if (!qdisc)
 645                 return NULL;
 646
 647         ops = qdisc->ops;
 648         if (!ops)
 649                 return NULL;
 650
 651         if (!ingress && !strcmp("ingress", ops->id))
 652                 return NULL;
 653
 654         cops = ops->cl_ops;
 655         if (!cops)
 656                 return NULL;
 657
 658         if (!cops->tcf_block)
 659                 return NULL;
 660
 661         return cops->tcf_block(qdisc,
 662                                ingress ? TC_H_MIN_INGRESS : TC_H_MIN_EGRESS,
 663                                NULL);
 664 }
 665
 666 static void tc_indr_block_get_and_cmd(struct net_device *dev,
 667                                       flow_indr_block_bind_cb_t *cb,
 668                                       void *cb_priv,
 669                                       enum flow_block_command command)
 670 {
 671         struct tcf_block *block;
 672
 673         block = tc_dev_block(dev, true);
 674         tc_indr_block_cmd(dev, block, cb, cb_priv, command, true);
 675
 676         block = tc_dev_block(dev, false);
 677         tc_indr_block_cmd(dev, block, cb, cb_priv, command, false);
 678 }
 679
 680 static void tc_indr_block_call(struct tcf_block *block,
 681                                struct net_device *dev,
 682                                struct tcf_block_ext_info *ei,
 683                                enum flow_block_command command,
 684                                struct netlink_ext_ack *extack)
 685 {
 686         struct flow_block_offload bo = {
 687                 .command        = command,
 688                 .binder_type    = ei->binder_type,
 689                 .net            = dev_net(dev),
 690                 .block          = &block->flow_block,
 691                 .block_shared   = tcf_block_shared(block),
 692                 .extack         = extack,
 693         };
 694         INIT_LIST_HEAD(&bo.cb_list);
 695
 696         flow_indr_block_call(dev, &bo, command);
 697         tcf_block_setup(block, &bo);
 698 }
 699
 700 static bool tcf_block_offload_in_use(struct tcf_block *block)
 701 {
 702         return atomic_read(&block->offloadcnt);
 703 }
 704
 705 static int tcf_block_offload_cmd(struct tcf_block *block,
 706                                  struct net_device *dev,
 707                                  struct tcf_block_ext_info *ei,
 708                                  enum flow_block_command command,
 709                                  struct netlink_ext_ack *extack)
 710 {
 711         struct flow_block_offload bo = {};
 712         int err;
 713
 714         bo.net = dev_net(dev);
 715         bo.command = command;
 716         bo.binder_type = ei->binder_type;
 717         bo.block = &block->flow_block;
 718         bo.block_shared = tcf_block_shared(block);
 719         bo.extack = extack;
 720         INIT_LIST_HEAD(&bo.cb_list);
 721
 722         err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 723         if (err < 0)
 724                 return err;
 725
 726         return tcf_block_setup(block, &bo);
 727 }
 728
 729 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
 730                                   struct tcf_block_ext_info *ei,
 731                                   struct netlink_ext_ack *extack)
 732 {
 733         struct net_device *dev = q->dev_queue->dev;
 734         int err;
 735
 736         down_write(&block->cb_lock);
 737         if (!dev->netdev_ops->ndo_setup_tc)
 738                 goto no_offload_dev_inc;
 739
 740         /* If tc offload feature is disabled and the block we try to bind
 741          * to already has some offloaded filters, forbid to bind.
 742          */
 743         if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) {
 744                 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
 745                 err = -EOPNOTSUPP;
 746                 goto err_unlock;
 747         }
 748
 749         err = tcf_block_offload_cmd(block, dev, ei, FLOW_BLOCK_BIND, extack);
 750         if (err == -EOPNOTSUPP)
 751                 goto no_offload_dev_inc;
 752         if (err)
 753                 goto err_unlock;
 754
 755         tc_indr_block_call(block, dev, ei, FLOW_BLOCK_BIND, extack);
 756         up_write(&block->cb_lock);
 757         return 0;
 758
 759 no_offload_dev_inc:
 760         if (tcf_block_offload_in_use(block)) {
 761                 err = -EOPNOTSUPP;
 762                 goto err_unlock;
 763         }
 764         err = 0;
 765         block->nooffloaddevcnt++;
 766         tc_indr_block_call(block, dev, ei, FLOW_BLOCK_BIND, extack);
 767 err_unlock:
 768         up_write(&block->cb_lock);
 769         return err;
 770 }
 771
 772 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
 773                                      struct tcf_block_ext_info *ei)
 774 {
 775         struct net_device *dev = q->dev_queue->dev;
 776         int err;
 777
 778         down_write(&block->cb_lock);
 779         tc_indr_block_call(block, dev, ei, FLOW_BLOCK_UNBIND, NULL);
 780
 781         if (!dev->netdev_ops->ndo_setup_tc)
 782                 goto no_offload_dev_dec;
 783         err = tcf_block_offload_cmd(block, dev, ei, FLOW_BLOCK_UNBIND, NULL);
 784         if (err == -EOPNOTSUPP)
 785                 goto no_offload_dev_dec;
 786         up_write(&block->cb_lock);
 787         return;
 788
 789 no_offload_dev_dec:
 790         WARN_ON(block->nooffloaddevcnt-- == 0);
 791         up_write(&block->cb_lock);
 792 }
 793
 794 static int
 795 tcf_chain0_head_change_cb_add(struct tcf_block *block,
 796                               struct tcf_block_ext_info *ei,
 797                               struct netlink_ext_ack *extack)
 798 {
 799         struct tcf_filter_chain_list_item *item;
 800         struct tcf_chain *chain0;
 801
 802         item = kmalloc(sizeof(*item), GFP_KERNEL);
 803         if (!item) {
 804                 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
 805                 return -ENOMEM;
 806         }
 807         item->chain_head_change = ei->chain_head_change;
 808         item->chain_head_change_priv = ei->chain_head_change_priv;
 809
 810         mutex_lock(&block->lock);
 811         chain0 = block->chain0.chain;
 812         if (chain0)
 813                 tcf_chain_hold(chain0);
 814         else
 815                 list_add(&item->list, &block->chain0.filter_chain_list);
 816         mutex_unlock(&block->lock);
 817
 818         if (chain0) {
 819                 struct tcf_proto *tp_head;
 820
 821                 mutex_lock(&chain0->filter_chain_lock);
 822
 823                 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
 824                 if (tp_head)
 825                         tcf_chain_head_change_item(item, tp_head);
 826
 827                 mutex_lock(&block->lock);
 828                 list_add(&item->list, &block->chain0.filter_chain_list);
 829                 mutex_unlock(&block->lock);
 830
 831                 mutex_unlock(&chain0->filter_chain_lock);
 832                 tcf_chain_put(chain0);
 833         }
 834
 835         return 0;
 836 }
 837
 838 static void
 839 tcf_chain0_head_change_cb_del(struct tcf_block *block,
 840                               struct tcf_block_ext_info *ei)
 841 {
 842         struct tcf_filter_chain_list_item *item;
 843
 844         mutex_lock(&block->lock);
 845         list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
 846                 if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
 847                     (item->chain_head_change == ei->chain_head_change &&
 848                      item->chain_head_change_priv == ei->chain_head_change_priv)) {
 849                         if (block->chain0.chain)
 850                                 tcf_chain_head_change_item(item, NULL);
 851                         list_del(&item->list);
 852                         mutex_unlock(&block->lock);
 853
 854                         kfree(item);
 855                         return;
 856                 }
 857         }
 858         mutex_unlock(&block->lock);
 859         WARN_ON(1);
 860 }
 861
 862 struct tcf_net {
 863         spinlock_t idr_lock; /* Protects idr */
 864         struct idr idr;
 865 };
 866
 867 static unsigned int tcf_net_id;
 868
 869 static int tcf_block_insert(struct tcf_block *block, struct net *net,
 870                             struct netlink_ext_ack *extack)
 871 {
 872         struct tcf_net *tn = net_generic(net, tcf_net_id);
 873         int err;
 874
 875         idr_preload(GFP_KERNEL);
 876         spin_lock(&tn->idr_lock);
 877         err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
 878                             GFP_NOWAIT);
 879         spin_unlock(&tn->idr_lock);
 880         idr_preload_end();
 881
 882         return err;
 883 }
 884
 885 static void tcf_block_remove(struct tcf_block *block, struct net *net)
 886 {
 887         struct tcf_net *tn = net_generic(net, tcf_net_id);
 888
 889         spin_lock(&tn->idr_lock);
 890         idr_remove(&tn->idr, block->index);
 891         spin_unlock(&tn->idr_lock);
 892 }
 893
 894 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
 895                                           u32 block_index,
 896                                           struct netlink_ext_ack *extack)
 897 {
 898         struct tcf_block *block;
 899
 900         block = kzalloc(sizeof(*block), GFP_KERNEL);
 901         if (!block) {
 902                 NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
 903                 return ERR_PTR(-ENOMEM);
 904         }
 905         mutex_init(&block->lock);
 906         mutex_init(&block->proto_destroy_lock);
 907         init_rwsem(&block->cb_lock);
 908         flow_block_init(&block->flow_block);
 909         INIT_LIST_HEAD(&block->chain_list);
 910         INIT_LIST_HEAD(&block->owner_list);
 911         INIT_LIST_HEAD(&block->chain0.filter_chain_list);
 912
 913         refcount_set(&block->refcnt, 1);
 914         block->net = net;
 915         block->index = block_index;
 916
 917         /* Don't store q pointer for blocks which are shared */
 918         if (!tcf_block_shared(block))
 919                 block->q = q;
 920         return block;
 921 }
 922
 923 static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
 924 {
 925         struct tcf_net *tn = net_generic(net, tcf_net_id);
 926
 927         return idr_find(&tn->idr, block_index);
 928 }
 929
 930 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
 931 {
 932         struct tcf_block *block;
 933
 934         rcu_read_lock();
 935         block = tcf_block_lookup(net, block_index);
 936         if (block && !refcount_inc_not_zero(&block->refcnt))
 937                 block = NULL;
 938         rcu_read_unlock();
 939
 940         return block;
 941 }
 942
 943 static struct tcf_chain *
 944 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
 945 {
 946         mutex_lock(&block->lock);
 947         if (chain)
 948                 chain = list_is_last(&chain->list, &block->chain_list) ?
 949                         NULL : list_next_entry(chain, list);
 950         else
 951                 chain = list_first_entry_or_null(&block->chain_list,
 952                                                  struct tcf_chain, list);
 953
 954         /* skip all action-only chains */
 955         while (chain && tcf_chain_held_by_acts_only(chain))
 956                 chain = list_is_last(&chain->list, &block->chain_list) ?
 957                         NULL : list_next_entry(chain, list);
 958
 959         if (chain)
 960                 tcf_chain_hold(chain);
 961         mutex_unlock(&block->lock);
 962
 963         return chain;
 964 }
 965
 966 /* Function to be used by all clients that want to iterate over all chains on
 967  * block. It properly obtains block->lock and takes reference to chain before
 968  * returning it. Users of this function must be tolerant to concurrent chain
 969  * insertion/deletion or ensure that no concurrent chain modification is
 970  * possible. Note that all netlink dump callbacks cannot guarantee to provide
 971  * consistent dump because rtnl lock is released each time skb is filled with
 972  * data and sent to user-space.
 973  */
 974
 975 struct tcf_chain *
 976 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
 977 {
 978         struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
 979
 980         if (chain)
 981                 tcf_chain_put(chain);
 982
 983         return chain_next;
 984 }
 985 EXPORT_SYMBOL(tcf_get_next_chain);
 986
 987 static struct tcf_proto *
 988 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
 989 {
 990         u32 prio = 0;
 991
 992         ASSERT_RTNL();
 993         mutex_lock(&chain->filter_chain_lock);
 994
 995         if (!tp) {
 996                 tp = tcf_chain_dereference(chain->filter_chain, chain);
 997         } else if (tcf_proto_is_deleting(tp)) {
 998                 /* 'deleting' flag is set and chain->filter_chain_lock was
 999                  * unlocked, which means next pointer could be invalid. Restart
1000                  * search.
1001                  */
1002                 prio = tp->prio + 1;
1003                 tp = tcf_chain_dereference(chain->filter_chain, chain);
1004
1005                 for (; tp; tp = tcf_chain_dereference(tp->next, chain))
1006                         if (!tp->deleting && tp->prio >= prio)
1007                                 break;
1008         } else {
1009                 tp = tcf_chain_dereference(tp->next, chain);
1010         }
1011
1012         if (tp)
1013                 tcf_proto_get(tp);
1014
1015         mutex_unlock(&chain->filter_chain_lock);
1016
1017         return tp;
1018 }
1019
1020 /* Function to be used by all clients that want to iterate over all tp's on
1021  * chain. Users of this function must be tolerant to concurrent tp
1022  * insertion/deletion or ensure that no concurrent chain modification is
1023  * possible. Note that all netlink dump callbacks cannot guarantee to provide
1024  * consistent dump because rtnl lock is released each time skb is filled with
1025  * data and sent to user-space.
1026  */
1027
1028 struct tcf_proto *
1029 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp,
1030                    bool rtnl_held)
1031 {
1032         struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
1033
1034         if (tp)
1035                 tcf_proto_put(tp, rtnl_held, NULL);
1036
1037         return tp_next;
1038 }
1039 EXPORT_SYMBOL(tcf_get_next_proto);
1040
1041 static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
1042 {
1043         struct tcf_chain *chain;
1044
1045         /* Last reference to block. At this point chains cannot be added or
1046          * removed concurrently.
1047          */
1048         for (chain = tcf_get_next_chain(block, NULL);
1049              chain;
1050              chain = tcf_get_next_chain(block, chain)) {
1051                 tcf_chain_put_explicitly_created(chain);
1052                 tcf_chain_flush(chain, rtnl_held);
1053         }
1054 }
1055
1056 /* Lookup Qdisc and increments its reference counter.
1057  * Set parent, if necessary.
1058  */
1059
1060 static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
1061                             u32 *parent, int ifindex, bool rtnl_held,
1062                             struct netlink_ext_ack *extack)
1063 {
1064         const struct Qdisc_class_ops *cops;
1065         struct net_device *dev;
1066         int err = 0;
1067
1068         if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1069                 return 0;
1070
1071         rcu_read_lock();
1072
1073         /* Find link */
1074         dev = dev_get_by_index_rcu(net, ifindex);
1075         if (!dev) {
1076                 rcu_read_unlock();
1077                 return -ENODEV;
1078         }
1079
1080         /* Find qdisc */
1081         if (!*parent) {
1082                 *q = dev->qdisc;
1083                 *parent = (*q)->handle;
1084         } else {
1085                 *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1086                 if (!*q) {
1087                         NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1088                         err = -EINVAL;
1089                         goto errout_rcu;
1090                 }
1091         }
1092
1093         *q = qdisc_refcount_inc_nz(*q);
1094         if (!*q) {
1095                 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1096                 err = -EINVAL;
1097                 goto errout_rcu;
1098         }
1099
1100         /* Is it classful? */
1101         cops = (*q)->ops->cl_ops;
1102         if (!cops) {
1103                 NL_SET_ERR_MSG(extack, "Qdisc not classful");
1104                 err = -EINVAL;
1105                 goto errout_qdisc;
1106         }
1107
1108         if (!cops->tcf_block) {
1109                 NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1110                 err = -EOPNOTSUPP;
1111                 goto errout_qdisc;
1112         }
1113
1114 errout_rcu:
1115         /* At this point we know that qdisc is not noop_qdisc,
1116          * which means that qdisc holds a reference to net_device
1117          * and we hold a reference to qdisc, so it is safe to release
1118          * rcu read lock.
1119          */
1120         rcu_read_unlock();
1121         return err;
1122
1123 errout_qdisc:
1124         rcu_read_unlock();
1125
1126         if (rtnl_held)
1127                 qdisc_put(*q);
1128         else
1129                 qdisc_put_unlocked(*q);
1130         *q = NULL;
1131
1132         return err;
1133 }
1134
1135 static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
1136                                int ifindex, struct netlink_ext_ack *extack)
1137 {
1138         if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1139                 return 0;
1140
1141         /* Do we search for filter, attached to class? */
1142         if (TC_H_MIN(parent)) {
1143                 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1144
1145                 *cl = cops->find(q, parent);
1146                 if (*cl == 0) {
1147                         NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1148                         return -ENOENT;
1149                 }
1150         }
1151
1152         return 0;
1153 }
1154
1155 static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
1156                                           unsigned long cl, int ifindex,
1157                                           u32 block_index,
1158                                           struct netlink_ext_ack *extack)
1159 {
1160         struct tcf_block *block;
1161
1162         if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1163                 block = tcf_block_refcnt_get(net, block_index);
1164                 if (!block) {
1165                         NL_SET_ERR_MSG(extack, "Block of given index was not found");
1166                         return ERR_PTR(-EINVAL);
1167                 }
1168         } else {
1169                 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1170
1171                 block = cops->tcf_block(q, cl, extack);
1172                 if (!block)
1173                         return ERR_PTR(-EINVAL);
1174
1175                 if (tcf_block_shared(block)) {
1176                         NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1177                         return ERR_PTR(-EOPNOTSUPP);
1178                 }
1179
1180                 /* Always take reference to block in order to support execution
1181                  * of rules update path of cls API without rtnl lock. Caller
1182                  * must release block when it is finished using it. 'if' block
1183                  * of this conditional obtain reference to block by calling
1184                  * tcf_block_refcnt_get().
1185                  */
1186                 refcount_inc(&block->refcnt);
1187         }
1188
1189         return block;
1190 }
1191
1192 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
1193                             struct tcf_block_ext_info *ei, bool rtnl_held)
1194 {
1195         if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
1196                 /* Flushing/putting all chains will cause the block to be
1197                  * deallocated when last chain is freed. However, if chain_list
1198                  * is empty, block has to be manually deallocated. After block
1199                  * reference counter reached 0, it is no longer possible to
1200                  * increment it or add new chains to block.
1201                  */
1202                 bool free_block = list_empty(&block->chain_list);
1203
1204                 mutex_unlock(&block->lock);
1205                 if (tcf_block_shared(block))
1206                         tcf_block_remove(block, block->net);
1207
1208                 if (q)
1209                         tcf_block_offload_unbind(block, q, ei);
1210
1211                 if (free_block)
1212                         tcf_block_destroy(block);
1213                 else
1214                         tcf_block_flush_all_chains(block, rtnl_held);
1215         } else if (q) {
1216                 tcf_block_offload_unbind(block, q, ei);
1217         }
1218 }
1219
1220 static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
1221 {
1222         __tcf_block_put(block, NULL, NULL, rtnl_held);
1223 }
1224
1225 /* Find tcf block.
1226  * Set q, parent, cl when appropriate.
1227  */
1228
1229 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
1230                                         u32 *parent, unsigned long *cl,
1231                                         int ifindex, u32 block_index,
1232                                         struct netlink_ext_ack *extack)
1233 {
1234         struct tcf_block *block;
1235         int err = 0;
1236
1237         ASSERT_RTNL();
1238
1239         err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack);
1240         if (err)
1241                 goto errout;
1242
1243         err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack);
1244         if (err)
1245                 goto errout_qdisc;
1246
1247         block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack);
1248         if (IS_ERR(block)) {
1249                 err = PTR_ERR(block);
1250                 goto errout_qdisc;
1251         }
1252
1253         return block;
1254
1255 errout_qdisc:
1256         if (*q)
1257                 qdisc_put(*q);
1258 errout:
1259         *q = NULL;
1260         return ERR_PTR(err);
1261 }
1262
1263 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
1264                               bool rtnl_held)
1265 {
1266         if (!IS_ERR_OR_NULL(block))
1267                 tcf_block_refcnt_put(block, rtnl_held);
1268
1269         if (q) {
1270                 if (rtnl_held)
1271                         qdisc_put(q);
1272                 else
1273                         qdisc_put_unlocked(q);
1274         }
1275 }
1276
1277 struct tcf_block_owner_item {
1278         struct list_head list;
1279         struct Qdisc *q;
1280         enum flow_block_binder_type binder_type;
1281 };
1282
1283 static void
1284 tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1285                                struct Qdisc *q,
1286                                enum flow_block_binder_type binder_type)
1287 {
1288         if (block->keep_dst &&
1289             binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1290             binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
1291                 netif_keep_dst(qdisc_dev(q));
1292 }
1293
1294 void tcf_block_netif_keep_dst(struct tcf_block *block)
1295 {
1296         struct tcf_block_owner_item *item;
1297
1298         block->keep_dst = true;
1299         list_for_each_entry(item, &block->owner_list, list)
1300                 tcf_block_owner_netif_keep_dst(block, item->q,
1301                                                item->binder_type);
1302 }
1303 EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1304
1305 static int tcf_block_owner_add(struct tcf_block *block,
1306                                struct Qdisc *q,
1307                                enum flow_block_binder_type binder_type)
1308 {
1309         struct tcf_block_owner_item *item;
1310
1311         item = kmalloc(sizeof(*item), GFP_KERNEL);
1312         if (!item)
1313                 return -ENOMEM;
1314         item->q = q;
1315         item->binder_type = binder_type;
1316         list_add(&item->list, &block->owner_list);
1317         return 0;
1318 }
1319
1320 static void tcf_block_owner_del(struct tcf_block *block,
1321                                 struct Qdisc *q,
1322                                 enum flow_block_binder_type binder_type)
1323 {
1324         struct tcf_block_owner_item *item;
1325
1326         list_for_each_entry(item, &block->owner_list, list) {
1327                 if (item->q == q && item->binder_type == binder_type) {
1328                         list_del(&item->list);
1329                         kfree(item);
1330                         return;
1331                 }
1332         }
1333         WARN_ON(1);
1334 }
1335
1336 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1337                       struct tcf_block_ext_info *ei,
1338                       struct netlink_ext_ack *extack)
1339 {
1340         struct net *net = qdisc_net(q);
1341         struct tcf_block *block = NULL;
1342         int err;
1343
1344         if (ei->block_index)
1345                 /* block_index not 0 means the shared block is requested */
1346                 block = tcf_block_refcnt_get(net, ei->block_index);
1347
1348         if (!block) {
1349                 block = tcf_block_create(net, q, ei->block_index, extack);
1350                 if (IS_ERR(block))
1351                         return PTR_ERR(block);
1352                 if (tcf_block_shared(block)) {
1353                         err = tcf_block_insert(block, net, extack);
1354                         if (err)
1355                                 goto err_block_insert;
1356                 }
1357         }
1358
1359         err = tcf_block_owner_add(block, q, ei->binder_type);
1360         if (err)
1361                 goto err_block_owner_add;
1362
1363         tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1364
1365         err = tcf_chain0_head_change_cb_add(block, ei, extack);
1366         if (err)
1367                 goto err_chain0_head_change_cb_add;
1368
1369         err = tcf_block_offload_bind(block, q, ei, extack);
1370         if (err)
1371                 goto err_block_offload_bind;
1372
1373         *p_block = block;
1374         return 0;
1375
1376 err_block_offload_bind:
1377         tcf_chain0_head_change_cb_del(block, ei);
1378 err_chain0_head_change_cb_add:
1379         tcf_block_owner_del(block, q, ei->binder_type);
1380 err_block_owner_add:
1381 err_block_insert:
1382         tcf_block_refcnt_put(block, true);
1383         return err;
1384 }
1385 EXPORT_SYMBOL(tcf_block_get_ext);
1386
1387 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1388 {
1389         struct tcf_proto __rcu **p_filter_chain = priv;
1390
1391         rcu_assign_pointer(*p_filter_chain, tp_head);
1392 }
1393
1394 int tcf_block_get(struct tcf_block **p_block,
1395                   struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1396                   struct netlink_ext_ack *extack)
1397 {
1398         struct tcf_block_ext_info ei = {
1399                 .chain_head_change = tcf_chain_head_change_dflt,
1400                 .chain_head_change_priv = p_filter_chain,
1401         };
1402
1403         WARN_ON(!p_filter_chain);
1404         return tcf_block_get_ext(p_block, q, &ei, extack);
1405 }
1406 EXPORT_SYMBOL(tcf_block_get);
1407
1408 /* XXX: Standalone actions are not allowed to jump to any chain, and bound
1409  * actions should be all removed after flushing.
1410  */
1411 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
1412                        struct tcf_block_ext_info *ei)
1413 {
1414         if (!block)
1415                 return;
1416         tcf_chain0_head_change_cb_del(block, ei);
1417         tcf_block_owner_del(block, q, ei->binder_type);
1418
1419         __tcf_block_put(block, q, ei, true);
1420 }
1421 EXPORT_SYMBOL(tcf_block_put_ext);
1422
1423 void tcf_block_put(struct tcf_block *block)
1424 {
1425         struct tcf_block_ext_info ei = {0, };
1426
1427         if (!block)
1428                 return;
1429         tcf_block_put_ext(block, block->q, &ei);
1430 }
1431
1432 EXPORT_SYMBOL(tcf_block_put);
1433
1434 static int
1435 tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
1436                             void *cb_priv, bool add, bool offload_in_use,
1437                             struct netlink_ext_ack *extack)
1438 {
1439         struct tcf_chain *chain, *chain_prev;
1440         struct tcf_proto *tp, *tp_prev;
1441         int err;
1442
1443         lockdep_assert_held(&block->cb_lock);
1444
1445         for (chain = __tcf_get_next_chain(block, NULL);
1446              chain;
1447              chain_prev = chain,
1448                      chain = __tcf_get_next_chain(block, chain),
1449                      tcf_chain_put(chain_prev)) {
1450                 for (tp = __tcf_get_next_proto(chain, NULL); tp;
1451                      tp_prev = tp,
1452                              tp = __tcf_get_next_proto(chain, tp),
1453                              tcf_proto_put(tp_prev, true, NULL)) {
1454                         if (tp->ops->reoffload) {
1455                                 err = tp->ops->reoffload(tp, add, cb, cb_priv,
1456                                                          extack);
1457                                 if (err && add)
1458                                         goto err_playback_remove;
1459                         } else if (add && offload_in_use) {
1460                                 err = -EOPNOTSUPP;
1461                                 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1462                                 goto err_playback_remove;
1463                         }
1464                 }
1465         }
1466
1467         return 0;
1468
1469 err_playback_remove:
1470         tcf_proto_put(tp, true, NULL);
1471         tcf_chain_put(chain);
1472         tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1473                                     extack);
1474         return err;
1475 }
1476
1477 static int tcf_block_bind(struct tcf_block *block,
1478                           struct flow_block_offload *bo)
1479 {
1480         struct flow_block_cb *block_cb, *next;
1481         int err, i = 0;
1482
1483         lockdep_assert_held(&block->cb_lock);
1484
1485         list_for_each_entry(block_cb, &bo->cb_list, list) {
1486                 err = tcf_block_playback_offloads(block, block_cb->cb,
1487                                                   block_cb->cb_priv, true,
1488                                                   tcf_block_offload_in_use(block),
1489                                                   bo->extack);
1490                 if (err)
1491                         goto err_unroll;
1492                 if (!bo->unlocked_driver_cb)
1493                         block->lockeddevcnt++;
1494
1495                 i++;
1496         }
1497         list_splice(&bo->cb_list, &block->flow_block.cb_list);
1498
1499         return 0;
1500
1501 err_unroll:
1502         list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1503                 if (i-- > 0) {
1504                         list_del(&block_cb->list);
1505                         tcf_block_playback_offloads(block, block_cb->cb,
1506                                                     block_cb->cb_priv, false,
1507                                                     tcf_block_offload_in_use(block),
1508                                                     NULL);
1509                         if (!bo->unlocked_driver_cb)
1510                                 block->lockeddevcnt--;
1511                 }
1512                 flow_block_cb_free(block_cb);
1513         }
1514
1515         return err;
1516 }
1517
1518 static void tcf_block_unbind(struct tcf_block *block,
1519                              struct flow_block_offload *bo)
1520 {
1521         struct flow_block_cb *block_cb, *next;
1522
1523         lockdep_assert_held(&block->cb_lock);
1524
1525         list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1526                 tcf_block_playback_offloads(block, block_cb->cb,
1527                                             block_cb->cb_priv, false,
1528                                             tcf_block_offload_in_use(block),
1529                                             NULL);
1530                 list_del(&block_cb->list);
1531                 flow_block_cb_free(block_cb);
1532                 if (!bo->unlocked_driver_cb)
1533                         block->lockeddevcnt--;
1534         }
1535 }
1536
1537 static int tcf_block_setup(struct tcf_block *block,
1538                            struct flow_block_offload *bo)
1539 {
1540         int err;
1541
1542         switch (bo->command) {
1543         case FLOW_BLOCK_BIND:
1544                 err = tcf_block_bind(block, bo);
1545                 break;
1546         case FLOW_BLOCK_UNBIND:
1547                 err = 0;
1548                 tcf_block_unbind(block, bo);
1549                 break;
1550         default:
1551                 WARN_ON_ONCE(1);
1552                 err = -EOPNOTSUPP;
1553         }
1554
1555         return err;
1556 }
1557
1558 /* Main classifier routine: scans classifier chain attached
1559  * to this qdisc, (optionally) tests for protocol and asks
1560  * specific classifiers.
1561  */
1562 int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1563                  struct tcf_result *res, bool compat_mode)
1564 {
1565 #ifdef CONFIG_NET_CLS_ACT
1566         const int max_reclassify_loop = 4;
1567         const struct tcf_proto *orig_tp = tp;
1568         const struct tcf_proto *first_tp;
1569         int limit = 0;
1570
1571 reclassify:
1572 #endif
1573         for (; tp; tp = rcu_dereference_bh(tp->next)) {
1574                 __be16 protocol = skb_protocol(skb, false);
1575                 int err;
1576
1577                 if (tp->protocol != protocol &&
1578                     tp->protocol != htons(ETH_P_ALL))
1579                         continue;
1580
1581                 err = tp->classify(skb, tp, res);
1582 #ifdef CONFIG_NET_CLS_ACT
1583                 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
1584                         first_tp = orig_tp;
1585                         goto reset;
1586                 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
1587                         first_tp = res->goto_tp;
1588
1589 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1590                         {
1591                                 struct tc_skb_ext *ext;
1592
1593                                 ext = skb_ext_add(skb, TC_SKB_EXT);
1594                                 if (WARN_ON_ONCE(!ext))
1595                                         return TC_ACT_SHOT;
1596
1597                                 ext->chain = err & TC_ACT_EXT_VAL_MASK;
1598                         }
1599 #endif
1600                         goto reset;
1601                 }
1602 #endif
1603                 if (err >= 0)
1604                         return err;
1605         }
1606
1607         return TC_ACT_UNSPEC; /* signal: continue lookup */
1608 #ifdef CONFIG_NET_CLS_ACT
1609 reset:
1610         if (unlikely(limit++ >= max_reclassify_loop)) {
1611                 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1612                                        tp->chain->block->index,
1613                                        tp->prio & 0xffff,
1614                                        ntohs(tp->protocol));
1615                 return TC_ACT_SHOT;
1616         }
1617
1618         tp = first_tp;
1619         goto reclassify;
1620 #endif
1621 }
1622 EXPORT_SYMBOL(tcf_classify);
1623
1624 struct tcf_chain_info {
1625         struct tcf_proto __rcu **pprev;
1626         struct tcf_proto __rcu *next;
1627 };
1628
1629 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1630                                            struct tcf_chain_info *chain_info)
1631 {
1632         return tcf_chain_dereference(*chain_info->pprev, chain);
1633 }
1634
1635 static int tcf_chain_tp_insert(struct tcf_chain *chain,
1636                                struct tcf_chain_info *chain_info,
1637                                struct tcf_proto *tp)
1638 {
1639         if (chain->flushing)
1640                 return -EAGAIN;
1641
1642         RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
1643         if (*chain_info->pprev == chain->filter_chain)
1644                 tcf_chain0_head_change(chain, tp);
1645         tcf_proto_get(tp);
1646         rcu_assign_pointer(*chain_info->pprev, tp);
1647
1648         return 0;
1649 }
1650
1651 static void tcf_chain_tp_remove(struct tcf_chain *chain,
1652                                 struct tcf_chain_info *chain_info,
1653                                 struct tcf_proto *tp)
1654 {
1655         struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
1656
1657         tcf_proto_mark_delete(tp);
1658         if (tp == chain->filter_chain)
1659                 tcf_chain0_head_change(chain, next);
1660         RCU_INIT_POINTER(*chain_info->pprev, next);
1661 }
1662
1663 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1664                                            struct tcf_chain_info *chain_info,
1665                                            u32 protocol, u32 prio,
1666                                            bool prio_allocate);
1667
1668 /* Try to insert new proto.
1669  * If proto with specified priority already exists, free new proto
1670  * and return existing one.
1671  */
1672
1673 static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1674                                                     struct tcf_proto *tp_new,
1675                                                     u32 protocol, u32 prio,
1676                                                     bool rtnl_held)
1677 {
1678         struct tcf_chain_info chain_info;
1679         struct tcf_proto *tp;
1680         int err = 0;
1681
1682         mutex_lock(&chain->filter_chain_lock);
1683
1684         if (tcf_proto_exists_destroying(chain, tp_new)) {
1685                 mutex_unlock(&chain->filter_chain_lock);
1686                 tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1687                 return ERR_PTR(-EAGAIN);
1688         }
1689
1690         tp = tcf_chain_tp_find(chain, &chain_info,
1691                                protocol, prio, false);
1692         if (!tp)
1693                 err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
1694         mutex_unlock(&chain->filter_chain_lock);
1695
1696         if (tp) {
1697                 tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1698                 tp_new = tp;
1699         } else if (err) {
1700                 tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1701                 tp_new = ERR_PTR(err);
1702         }
1703
1704         return tp_new;
1705 }
1706
1707 static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
1708                                       struct tcf_proto *tp, bool rtnl_held,
1709                                       struct netlink_ext_ack *extack)
1710 {
1711         struct tcf_chain_info chain_info;
1712         struct tcf_proto *tp_iter;
1713         struct tcf_proto **pprev;
1714         struct tcf_proto *next;
1715
1716         mutex_lock(&chain->filter_chain_lock);
1717
1718         /* Atomically find and remove tp from chain. */
1719         for (pprev = &chain->filter_chain;
1720              (tp_iter = tcf_chain_dereference(*pprev, chain));
1721              pprev = &tp_iter->next) {
1722                 if (tp_iter == tp) {
1723                         chain_info.pprev = pprev;
1724                         chain_info.next = tp_iter->next;
1725                         WARN_ON(tp_iter->deleting);
1726                         break;
1727                 }
1728         }
1729         /* Verify that tp still exists and no new filters were inserted
1730          * concurrently.
1731          * Mark tp for deletion if it is empty.
1732          */
1733         if (!tp_iter || !tcf_proto_check_delete(tp)) {
1734                 mutex_unlock(&chain->filter_chain_lock);
1735                 return;
1736         }
1737
1738         tcf_proto_signal_destroying(chain, tp);
1739         next = tcf_chain_dereference(chain_info.next, chain);
1740         if (tp == chain->filter_chain)
1741                 tcf_chain0_head_change(chain, next);
1742         RCU_INIT_POINTER(*chain_info.pprev, next);
1743         mutex_unlock(&chain->filter_chain_lock);
1744
1745         tcf_proto_put(tp, rtnl_held, extack);
1746 }
1747
1748 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1749                                            struct tcf_chain_info *chain_info,
1750                                            u32 protocol, u32 prio,
1751                                            bool prio_allocate)
1752 {
1753         struct tcf_proto **pprev;
1754         struct tcf_proto *tp;
1755
1756         /* Check the chain for existence of proto-tcf with this priority */
1757         for (pprev = &chain->filter_chain;
1758              (tp = tcf_chain_dereference(*pprev, chain));
1759              pprev = &tp->next) {
1760                 if (tp->prio >= prio) {
1761                         if (tp->prio == prio) {
1762                                 if (prio_allocate ||
1763                                     (tp->protocol != protocol && protocol))
1764                                         return ERR_PTR(-EINVAL);
1765                         } else {
1766                                 tp = NULL;
1767                         }
1768                         break;
1769                 }
1770         }
1771         chain_info->pprev = pprev;
1772         if (tp) {
1773                 chain_info->next = tp->next;
1774                 tcf_proto_get(tp);
1775         } else {
1776                 chain_info->next = NULL;
1777         }
1778         return tp;
1779 }
1780
1781 static int tcf_fill_node(struct net *net, struct sk_buff *skb,
1782                          struct tcf_proto *tp, struct tcf_block *block,
1783                          struct Qdisc *q, u32 parent, void *fh,
1784                          u32 portid, u32 seq, u16 flags, int event,
1785                          bool rtnl_held)
1786 {
1787         struct tcmsg *tcm;
1788         struct nlmsghdr  *nlh;
1789         unsigned char *b = skb_tail_pointer(skb);
1790
1791         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1792         if (!nlh)
1793                 goto out_nlmsg_trim;
1794         tcm = nlmsg_data(nlh);
1795         tcm->tcm_family = AF_UNSPEC;
1796         tcm->tcm__pad1 = 0;
1797         tcm->tcm__pad2 = 0;
1798         if (q) {
1799                 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1800                 tcm->tcm_parent = parent;
1801         } else {
1802                 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1803                 tcm->tcm_block_index = block->index;
1804         }
1805         tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1806         if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1807                 goto nla_put_failure;
1808         if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1809                 goto nla_put_failure;
1810         if (!fh) {
1811                 tcm->tcm_handle = 0;
1812         } else {
1813                 if (tp->ops->dump &&
1814                     tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
1815                         goto nla_put_failure;
1816         }
1817         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1818         return skb->len;
1819
1820 out_nlmsg_trim:
1821 nla_put_failure:
1822         nlmsg_trim(skb, b);
1823         return -1;
1824 }
1825
1826 static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1827                           struct nlmsghdr *n, struct tcf_proto *tp,
1828                           struct tcf_block *block, struct Qdisc *q,
1829                           u32 parent, void *fh, int event, bool unicast,
1830                           bool rtnl_held)
1831 {
1832         struct sk_buff *skb;
1833         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1834         int err = 0;
1835
1836         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1837         if (!skb)
1838                 return -ENOBUFS;
1839
1840         if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1841                           n->nlmsg_seq, n->nlmsg_flags, event,
1842                           rtnl_held) <= 0) {
1843                 kfree_skb(skb);
1844                 return -EINVAL;
1845         }
1846
1847         if (unicast)
1848                 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1849         else
1850                 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1851                                      n->nlmsg_flags & NLM_F_ECHO);
1852
1853         if (err > 0)
1854                 err = 0;
1855         return err;
1856 }
1857
1858 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1859                               struct nlmsghdr *n, struct tcf_proto *tp,
1860                               struct tcf_block *block, struct Qdisc *q,
1861                               u32 parent, void *fh, bool unicast, bool *last,
1862                               bool rtnl_held, struct netlink_ext_ack *extack)
1863 {
1864         struct sk_buff *skb;
1865         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1866         int err;
1867
1868         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1869         if (!skb)
1870                 return -ENOBUFS;
1871
1872         if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1873                           n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER,
1874                           rtnl_held) <= 0) {
1875                 NL_SET_ERR_MSG(extack, "Failed to build del event notification");
1876                 kfree_skb(skb);
1877                 return -EINVAL;
1878         }
1879
1880         err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
1881         if (err) {
1882                 kfree_skb(skb);
1883                 return err;
1884         }
1885
1886         if (unicast)
1887                 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1888         else
1889                 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1890                                      n->nlmsg_flags & NLM_F_ECHO);
1891         if (err < 0)
1892                 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
1893
1894         if (err > 0)
1895                 err = 0;
1896         return err;
1897 }
1898
1899 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
1900                                  struct tcf_block *block, struct Qdisc *q,
1901                                  u32 parent, struct nlmsghdr *n,
1902                                  struct tcf_chain *chain, int event,
1903                                  bool rtnl_held)
1904 {
1905         struct tcf_proto *tp;
1906
1907         for (tp = tcf_get_next_proto(chain, NULL, rtnl_held);
1908              tp; tp = tcf_get_next_proto(chain, tp, rtnl_held))
1909                 tfilter_notify(net, oskb, n, tp, block,
1910                                q, parent, NULL, event, false, rtnl_held);
1911 }
1912
1913 static void tfilter_put(struct tcf_proto *tp, void *fh)
1914 {
1915         if (tp->ops->put && fh)
1916                 tp->ops->put(tp, fh);
1917 }
1918
1919 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1920                           struct netlink_ext_ack *extack)
1921 {
1922         struct net *net = sock_net(skb->sk);
1923         struct nlattr *tca[TCA_MAX + 1];
1924         char name[IFNAMSIZ];
1925         struct tcmsg *t;
1926         u32 protocol;
1927         u32 prio;
1928         bool prio_allocate;
1929         u32 parent;
1930         u32 chain_index;
1931         struct Qdisc *q;
1932         struct tcf_chain_info chain_info;
1933         struct tcf_chain *chain;
1934         struct tcf_block *block;
1935         struct tcf_proto *tp;
1936         unsigned long cl;
1937         void *fh;
1938         int err;
1939         int tp_created;
1940         bool rtnl_held = false;
1941
1942         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1943                 return -EPERM;
1944
1945 replay:
1946         tp_created = 0;
1947
1948         err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
1949                                      rtm_tca_policy, extack);
1950         if (err < 0)
1951                 return err;
1952
1953         t = nlmsg_data(n);
1954         protocol = TC_H_MIN(t->tcm_info);
1955         prio = TC_H_MAJ(t->tcm_info);
1956         prio_allocate = false;
1957         parent = t->tcm_parent;
1958         tp = NULL;
1959         cl = 0;
1960         block = NULL;
1961         q = NULL;
1962         chain = NULL;
1963
1964         if (prio == 0) {
1965                 /* If no priority is provided by the user,
1966                  * we allocate one.
1967                  */
1968                 if (n->nlmsg_flags & NLM_F_CREATE) {
1969                         prio = TC_H_MAKE(0x80000000U, 0U);
1970                         prio_allocate = true;
1971                 } else {
1972                         NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1973                         return -ENOENT;
1974                 }
1975         }
1976
1977         /* Find head of filter chain. */
1978
1979         err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
1980         if (err)
1981                 return err;
1982
1983         if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
1984                 NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
1985                 err = -EINVAL;
1986                 goto errout;
1987         }
1988
1989         /* Take rtnl mutex if rtnl_held was set to true on previous iteration,
1990          * block is shared (no qdisc found), qdisc is not unlocked, classifier
1991          * type is not specified, classifier is not unlocked.
1992          */
1993         if (rtnl_held ||
1994             (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
1995             !tcf_proto_is_unlocked(name)) {
1996                 rtnl_held = true;
1997                 rtnl_lock();
1998         }
1999
2000         err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2001         if (err)
2002                 goto errout;
2003
2004         block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2005                                  extack);
2006         if (IS_ERR(block)) {
2007                 err = PTR_ERR(block);
2008                 goto errout;
2009         }
2010         block->classid = parent;
2011
2012         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2013         if (chain_index > TC_ACT_EXT_VAL_MASK) {
2014                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2015                 err = -EINVAL;
2016                 goto errout;
2017         }
2018         chain = tcf_chain_get(block, chain_index, true);
2019         if (!chain) {
2020                 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
2021                 err = -ENOMEM;
2022                 goto errout;
2023         }
2024
2025         mutex_lock(&chain->filter_chain_lock);
2026         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2027                                prio, prio_allocate);
2028         if (IS_ERR(tp)) {
2029                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2030                 err = PTR_ERR(tp);
2031                 goto errout_locked;
2032         }
2033
2034         if (tp == NULL) {
2035                 struct tcf_proto *tp_new = NULL;
2036
2037                 if (chain->flushing) {
2038                         err = -EAGAIN;
2039                         goto errout_locked;
2040                 }
2041
2042                 /* Proto-tcf does not exist, create new one */
2043
2044                 if (tca[TCA_KIND] == NULL || !protocol) {
2045                         NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
2046                         err = -EINVAL;
2047                         goto errout_locked;
2048                 }
2049
2050                 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2051                         NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2052                         err = -ENOENT;
2053                         goto errout_locked;
2054                 }
2055
2056                 if (prio_allocate)
2057                         prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
2058                                                                &chain_info));
2059
2060                 mutex_unlock(&chain->filter_chain_lock);
2061                 tp_new = tcf_proto_create(name, protocol, prio, chain,
2062                                           rtnl_held, extack);
2063                 if (IS_ERR(tp_new)) {
2064                         err = PTR_ERR(tp_new);
2065                         goto errout_tp;
2066                 }
2067
2068                 tp_created = 1;
2069                 tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2070                                                 rtnl_held);
2071                 if (IS_ERR(tp)) {
2072                         err = PTR_ERR(tp);
2073                         goto errout_tp;
2074                 }
2075         } else {
2076                 mutex_unlock(&chain->filter_chain_lock);
2077         }
2078
2079         if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2080                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2081                 err = -EINVAL;
2082                 goto errout;
2083         }
2084
2085         fh = tp->ops->get(tp, t->tcm_handle);
2086
2087         if (!fh) {
2088                 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2089                         NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2090                         err = -ENOENT;
2091                         goto errout;
2092                 }
2093         } else if (n->nlmsg_flags & NLM_F_EXCL) {
2094                 tfilter_put(tp, fh);
2095                 NL_SET_ERR_MSG(extack, "Filter already exists");
2096                 err = -EEXIST;
2097                 goto errout;
2098         }
2099
2100         if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2101                 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
2102                 err = -EINVAL;
2103                 goto errout;
2104         }
2105
2106         err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
2107                               n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
2108                               rtnl_held, extack);
2109         if (err == 0) {
2110                 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2111                                RTM_NEWTFILTER, false, rtnl_held);
2112                 tfilter_put(tp, fh);
2113                 /* q pointer is NULL for shared blocks */
2114                 if (q)
2115                         q->flags &= ~TCQ_F_CAN_BYPASS;
2116         }
2117
2118 errout:
2119         if (err && tp_created)
2120                 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
2121 errout_tp:
2122         if (chain) {
2123                 if (tp && !IS_ERR(tp))
2124                         tcf_proto_put(tp, rtnl_held, NULL);
2125                 if (!tp_created)
2126                         tcf_chain_put(chain);
2127         }
2128         tcf_block_release(q, block, rtnl_held);
2129
2130         if (rtnl_held)
2131                 rtnl_unlock();
2132
2133         if (err == -EAGAIN) {
2134                 /* Take rtnl lock in case EAGAIN is caused by concurrent flush
2135                  * of target chain.
2136                  */
2137                 rtnl_held = true;
2138                 /* Replay the request. */
2139                 goto replay;
2140         }
2141         return err;
2142
2143 errout_locked:
2144         mutex_unlock(&chain->filter_chain_lock);
2145         goto errout;
2146 }
2147
2148 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2149                           struct netlink_ext_ack *extack)
2150 {
2151         struct net *net = sock_net(skb->sk);
2152         struct nlattr *tca[TCA_MAX + 1];
2153         char name[IFNAMSIZ];
2154         struct tcmsg *t;
2155         u32 protocol;
2156         u32 prio;
2157         u32 parent;
2158         u32 chain_index;
2159         struct Qdisc *q = NULL;
2160         struct tcf_chain_info chain_info;
2161         struct tcf_chain *chain = NULL;
2162         struct tcf_block *block = NULL;
2163         struct tcf_proto *tp = NULL;
2164         unsigned long cl = 0;
2165         void *fh = NULL;
2166         int err;
2167         bool rtnl_held = false;
2168
2169         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2170                 return -EPERM;
2171
2172         err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2173                                      rtm_tca_policy, extack);
2174         if (err < 0)
2175                 return err;
2176
2177         t = nlmsg_data(n);
2178         protocol = TC_H_MIN(t->tcm_info);
2179         prio = TC_H_MAJ(t->tcm_info);
2180         parent = t->tcm_parent;
2181
2182         if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
2183                 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
2184                 return -ENOENT;
2185         }
2186
2187         /* Find head of filter chain. */
2188
2189         err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2190         if (err)
2191                 return err;
2192
2193         if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2194                 NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2195                 err = -EINVAL;
2196                 goto errout;
2197         }
2198         /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
2199          * found), qdisc is not unlocked, classifier type is not specified,
2200          * classifier is not unlocked.
2201          */
2202         if (!prio ||
2203             (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2204             !tcf_proto_is_unlocked(name)) {
2205                 rtnl_held = true;
2206                 rtnl_lock();
2207         }
2208
2209         err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2210         if (err)
2211                 goto errout;
2212
2213         block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2214                                  extack);
2215         if (IS_ERR(block)) {
2216                 err = PTR_ERR(block);
2217                 goto errout;
2218         }
2219
2220         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2221         if (chain_index > TC_ACT_EXT_VAL_MASK) {
2222                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2223                 err = -EINVAL;
2224                 goto errout;
2225         }
2226         chain = tcf_chain_get(block, chain_index, false);
2227         if (!chain) {
2228                 /* User requested flush on non-existent chain. Nothing to do,
2229                  * so just return success.
2230                  */
2231                 if (prio == 0) {
2232                         err = 0;
2233                         goto errout;
2234                 }
2235                 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2236                 err = -ENOENT;
2237                 goto errout;
2238         }
2239
2240         if (prio == 0) {
2241                 tfilter_notify_chain(net, skb, block, q, parent, n,
2242                                      chain, RTM_DELTFILTER, rtnl_held);
2243                 tcf_chain_flush(chain, rtnl_held);
2244                 err = 0;
2245                 goto errout;
2246         }
2247
2248         mutex_lock(&chain->filter_chain_lock);
2249         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2250                                prio, false);
2251         if (!tp || IS_ERR(tp)) {
2252                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2253                 err = tp ? PTR_ERR(tp) : -ENOENT;
2254                 goto errout_locked;
2255         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2256                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2257                 err = -EINVAL;
2258                 goto errout_locked;
2259         } else if (t->tcm_handle == 0) {
2260                 tcf_proto_signal_destroying(chain, tp);
2261                 tcf_chain_tp_remove(chain, &chain_info, tp);
2262                 mutex_unlock(&chain->filter_chain_lock);
2263
2264                 tcf_proto_put(tp, rtnl_held, NULL);
2265                 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2266                                RTM_DELTFILTER, false, rtnl_held);
2267                 err = 0;
2268                 goto errout;
2269         }
2270         mutex_unlock(&chain->filter_chain_lock);
2271
2272         fh = tp->ops->get(tp, t->tcm_handle);
2273
2274         if (!fh) {
2275                 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2276                 err = -ENOENT;
2277         } else {
2278                 bool last;
2279
2280                 err = tfilter_del_notify(net, skb, n, tp, block,
2281                                          q, parent, fh, false, &last,
2282                                          rtnl_held, extack);
2283
2284                 if (err)
2285                         goto errout;
2286                 if (last)
2287                         tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
2288         }
2289
2290 errout:
2291         if (chain) {
2292                 if (tp && !IS_ERR(tp))
2293                         tcf_proto_put(tp, rtnl_held, NULL);
2294                 tcf_chain_put(chain);
2295         }
2296         tcf_block_release(q, block, rtnl_held);
2297
2298         if (rtnl_held)
2299                 rtnl_unlock();
2300
2301         return err;
2302
2303 errout_locked:
2304         mutex_unlock(&chain->filter_chain_lock);
2305         goto errout;
2306 }
2307
2308 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2309                           struct netlink_ext_ack *extack)
2310 {
2311         struct net *net = sock_net(skb->sk);
2312         struct nlattr *tca[TCA_MAX + 1];
2313         char name[IFNAMSIZ];
2314         struct tcmsg *t;
2315         u32 protocol;
2316         u32 prio;
2317         u32 parent;
2318         u32 chain_index;
2319         struct Qdisc *q = NULL;
2320         struct tcf_chain_info chain_info;
2321         struct tcf_chain *chain = NULL;
2322         struct tcf_block *block = NULL;
2323         struct tcf_proto *tp = NULL;
2324         unsigned long cl = 0;
2325         void *fh = NULL;
2326         int err;
2327         bool rtnl_held = false;
2328
2329         err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2330                                      rtm_tca_policy, extack);
2331         if (err < 0)
2332                 return err;
2333
2334         t = nlmsg_data(n);
2335         protocol = TC_H_MIN(t->tcm_info);
2336         prio = TC_H_MAJ(t->tcm_info);
2337         parent = t->tcm_parent;
2338
2339         if (prio == 0) {
2340                 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2341                 return -ENOENT;
2342         }
2343
2344         /* Find head of filter chain. */
2345
2346         err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2347         if (err)
2348                 return err;
2349
2350         if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2351                 NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2352                 err = -EINVAL;
2353                 goto errout;
2354         }
2355         /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
2356          * unlocked, classifier type is not specified, classifier is not
2357          * unlocked.
2358          */
2359         if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2360             !tcf_proto_is_unlocked(name)) {
2361                 rtnl_held = true;
2362                 rtnl_lock();
2363         }
2364
2365         err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2366         if (err)
2367                 goto errout;
2368
2369         block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2370                                  extack);
2371         if (IS_ERR(block)) {
2372                 err = PTR_ERR(block);
2373                 goto errout;
2374         }
2375
2376         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2377         if (chain_index > TC_ACT_EXT_VAL_MASK) {
2378                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2379                 err = -EINVAL;
2380                 goto errout;
2381         }
2382         chain = tcf_chain_get(block, chain_index, false);
2383         if (!chain) {
2384                 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2385                 err = -EINVAL;
2386                 goto errout;
2387         }
2388
2389         mutex_lock(&chain->filter_chain_lock);
2390         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2391                                prio, false);
2392         mutex_unlock(&chain->filter_chain_lock);
2393         if (!tp || IS_ERR(tp)) {
2394                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2395                 err = tp ? PTR_ERR(tp) : -ENOENT;
2396                 goto errout;
2397         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2398                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2399                 err = -EINVAL;
2400                 goto errout;
2401         }
2402
2403         fh = tp->ops->get(tp, t->tcm_handle);
2404
2405         if (!fh) {
2406                 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2407                 err = -ENOENT;
2408         } else {
2409                 err = tfilter_notify(net, skb, n, tp, block, q, parent,
2410                                      fh, RTM_NEWTFILTER, true, rtnl_held);
2411                 if (err < 0)
2412                         NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
2413         }
2414
2415         tfilter_put(tp, fh);
2416 errout:
2417         if (chain) {
2418                 if (tp && !IS_ERR(tp))
2419                         tcf_proto_put(tp, rtnl_held, NULL);
2420                 tcf_chain_put(chain);
2421         }
2422         tcf_block_release(q, block, rtnl_held);
2423
2424         if (rtnl_held)
2425                 rtnl_unlock();
2426
2427         return err;
2428 }
2429
2430 struct tcf_dump_args {
2431         struct tcf_walker w;
2432         struct sk_buff *skb;
2433         struct netlink_callback *cb;
2434         struct tcf_block *block;
2435         struct Qdisc *q;
2436         u32 parent;
2437 };
2438
2439 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
2440 {
2441         struct tcf_dump_args *a = (void *)arg;
2442         struct net *net = sock_net(a->skb->sk);
2443
2444         return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
2445                              n, NETLINK_CB(a->cb->skb).portid,
2446                              a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2447                              RTM_NEWTFILTER, true);
2448 }
2449
2450 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2451                            struct sk_buff *skb, struct netlink_callback *cb,
2452                            long index_start, long *p_index)
2453 {
2454         struct net *net = sock_net(skb->sk);
2455         struct tcf_block *block = chain->block;
2456         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2457         struct tcf_proto *tp, *tp_prev;
2458         struct tcf_dump_args arg;
2459
2460         for (tp = __tcf_get_next_proto(chain, NULL);
2461              tp;
2462              tp_prev = tp,
2463                      tp = __tcf_get_next_proto(chain, tp),
2464                      tcf_proto_put(tp_prev, true, NULL),
2465                      (*p_index)++) {
2466                 if (*p_index < index_start)
2467                         continue;
2468                 if (TC_H_MAJ(tcm->tcm_info) &&
2469                     TC_H_MAJ(tcm->tcm_info) != tp->prio)
2470                         continue;
2471                 if (TC_H_MIN(tcm->tcm_info) &&
2472                     TC_H_MIN(tcm->tcm_info) != tp->protocol)
2473                         continue;
2474                 if (*p_index > index_start)
2475                         memset(&cb->args[1], 0,
2476                                sizeof(cb->args) - sizeof(cb->args[0]));
2477                 if (cb->args[1] == 0) {
2478                         if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
2479                                           NETLINK_CB(cb->skb).portid,
2480                                           cb->nlh->nlmsg_seq, NLM_F_MULTI,
2481                                           RTM_NEWTFILTER, true) <= 0)
2482                                 goto errout;
2483                         cb->args[1] = 1;
2484                 }
2485                 if (!tp->ops->walk)
2486                         continue;
2487                 arg.w.fn = tcf_node_dump;
2488                 arg.skb = skb;
2489                 arg.cb = cb;
2490                 arg.block = block;
2491                 arg.q = q;
2492                 arg.parent = parent;
2493                 arg.w.stop = 0;
2494                 arg.w.skip = cb->args[1] - 1;
2495                 arg.w.count = 0;
2496                 arg.w.cookie = cb->args[2];
2497                 tp->ops->walk(tp, &arg.w, true);
2498                 cb->args[2] = arg.w.cookie;
2499                 cb->args[1] = arg.w.count + 1;
2500                 if (arg.w.stop)
2501                         goto errout;
2502         }
2503         return true;
2504
2505 errout:
2506         tcf_proto_put(tp, true, NULL);
2507         return false;
2508 }
2509
2510 /* called with RTNL */
2511 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2512 {
2513         struct tcf_chain *chain, *chain_prev;
2514         struct net *net = sock_net(skb->sk);
2515         struct nlattr *tca[TCA_MAX + 1];
2516         struct Qdisc *q = NULL;
2517         struct tcf_block *block;
2518         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2519         long index_start;
2520         long index;
2521         u32 parent;
2522         int err;
2523
2524         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2525                 return skb->len;
2526
2527         err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2528                                      NULL, cb->extack);
2529         if (err)
2530                 return err;
2531
2532         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2533                 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2534                 if (!block)
2535                         goto out;
2536                 /* If we work with block index, q is NULL and parent value
2537                  * will never be used in the following code. The check
2538                  * in tcf_fill_node prevents it. However, compiler does not
2539                  * see that far, so set parent to zero to silence the warning
2540                  * about parent being uninitialized.
2541                  */
2542                 parent = 0;
2543         } else {
2544                 const struct Qdisc_class_ops *cops;
2545                 struct net_device *dev;
2546                 unsigned long cl = 0;
2547
2548                 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2549                 if (!dev)
2550                         return skb->len;
2551
2552                 parent = tcm->tcm_parent;
2553                 if (!parent)
2554                         q = dev->qdisc;
2555                 else
2556                         q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2557                 if (!q)
2558                         goto out;
2559                 cops = q->ops->cl_ops;
2560                 if (!cops)
2561                         goto out;
2562                 if (!cops->tcf_block)
2563                         goto out;
2564                 if (TC_H_MIN(tcm->tcm_parent)) {
2565                         cl = cops->find(q, tcm->tcm_parent);
2566                         if (cl == 0)
2567                                 goto out;
2568                 }
2569                 block = cops->tcf_block(q, cl, NULL);
2570                 if (!block)
2571                         goto out;
2572                 parent = block->classid;
2573                 if (tcf_block_shared(block))
2574                         q = NULL;
2575         }
2576
2577         index_start = cb->args[0];
2578         index = 0;
2579
2580         for (chain = __tcf_get_next_chain(block, NULL);
2581              chain;
2582              chain_prev = chain,
2583                      chain = __tcf_get_next_chain(block, chain),
2584                      tcf_chain_put(chain_prev)) {
2585                 if (tca[TCA_CHAIN] &&
2586                     nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2587                         continue;
2588                 if (!tcf_chain_dump(chain, q, parent, skb, cb,
2589                                     index_start, &index)) {
2590                         tcf_chain_put(chain);
2591                         err = -EMSGSIZE;
2592                         break;
2593                 }
2594         }
2595
2596         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2597                 tcf_block_refcnt_put(block, true);
2598         cb->args[0] = index;
2599
2600 out:
2601         /* If we did no progress, the error (EMSGSIZE) is real */
2602         if (skb->len == 0 && err)
2603                 return err;
2604         return skb->len;
2605 }
2606
2607 static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2608                               void *tmplt_priv, u32 chain_index,
2609                               struct net *net, struct sk_buff *skb,
2610                               struct tcf_block *block,
2611                               u32 portid, u32 seq, u16 flags, int event)
2612 {
2613         unsigned char *b = skb_tail_pointer(skb);
2614         const struct tcf_proto_ops *ops;
2615         struct nlmsghdr *nlh;
2616         struct tcmsg *tcm;
2617         void *priv;
2618
2619         ops = tmplt_ops;
2620         priv = tmplt_priv;
2621
2622         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2623         if (!nlh)
2624                 goto out_nlmsg_trim;
2625         tcm = nlmsg_data(nlh);
2626         tcm->tcm_family = AF_UNSPEC;
2627         tcm->tcm__pad1 = 0;
2628         tcm->tcm__pad2 = 0;
2629         tcm->tcm_handle = 0;
2630         if (block->q) {
2631                 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2632                 tcm->tcm_parent = block->q->handle;
2633         } else {
2634                 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2635                 tcm->tcm_block_index = block->index;
2636         }
2637
2638         if (nla_put_u32(skb, TCA_CHAIN, chain_index))
2639                 goto nla_put_failure;
2640
2641         if (ops) {
2642                 if (nla_put_string(skb, TCA_KIND, ops->kind))
2643                         goto nla_put_failure;
2644                 if (ops->tmplt_dump(skb, net, priv) < 0)
2645                         goto nla_put_failure;
2646         }
2647
2648         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2649         return skb->len;
2650
2651 out_nlmsg_trim:
2652 nla_put_failure:
2653         nlmsg_trim(skb, b);
2654         return -EMSGSIZE;
2655 }
2656
2657 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2658                            u32 seq, u16 flags, int event, bool unicast)
2659 {
2660         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2661         struct tcf_block *block = chain->block;
2662         struct net *net = block->net;
2663         struct sk_buff *skb;
2664         int err = 0;
2665
2666         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2667         if (!skb)
2668                 return -ENOBUFS;
2669
2670         if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2671                                chain->index, net, skb, block, portid,
2672                                seq, flags, event) <= 0) {
2673                 kfree_skb(skb);
2674                 return -EINVAL;
2675         }
2676
2677         if (unicast)
2678                 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2679         else
2680                 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
2681                                      flags & NLM_F_ECHO);
2682
2683         if (err > 0)
2684                 err = 0;
2685         return err;
2686 }
2687
2688 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
2689                                   void *tmplt_priv, u32 chain_index,
2690                                   struct tcf_block *block, struct sk_buff *oskb,
2691                                   u32 seq, u16 flags, bool unicast)
2692 {
2693         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2694         struct net *net = block->net;
2695         struct sk_buff *skb;
2696
2697         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2698         if (!skb)
2699                 return -ENOBUFS;
2700
2701         if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
2702                                block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
2703                 kfree_skb(skb);
2704                 return -EINVAL;
2705         }
2706
2707         if (unicast)
2708                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2709
2710         return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2711 }
2712
2713 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2714                               struct nlattr **tca,
2715                               struct netlink_ext_ack *extack)
2716 {
2717         const struct tcf_proto_ops *ops;
2718         char name[IFNAMSIZ];
2719         void *tmplt_priv;
2720
2721         /* If kind is not set, user did not specify template. */
2722         if (!tca[TCA_KIND])
2723                 return 0;
2724
2725         if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2726                 NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
2727                 return -EINVAL;
2728         }
2729
2730         ops = tcf_proto_lookup_ops(name, true, extack);
2731         if (IS_ERR(ops))
2732                 return PTR_ERR(ops);
2733         if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2734                 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2735                 return -EOPNOTSUPP;
2736         }
2737
2738         tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2739         if (IS_ERR(tmplt_priv)) {
2740                 module_put(ops->owner);
2741                 return PTR_ERR(tmplt_priv);
2742         }
2743         chain->tmplt_ops = ops;
2744         chain->tmplt_priv = tmplt_priv;
2745         return 0;
2746 }
2747
2748 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
2749                                void *tmplt_priv)
2750 {
2751         /* If template ops are set, no work to do for us. */
2752         if (!tmplt_ops)
2753                 return;
2754
2755         tmplt_ops->tmplt_destroy(tmplt_priv);
2756         module_put(tmplt_ops->owner);
2757 }
2758
2759 /* Add/delete/get a chain */
2760
2761 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2762                         struct netlink_ext_ack *extack)
2763 {
2764         struct net *net = sock_net(skb->sk);
2765         struct nlattr *tca[TCA_MAX + 1];
2766         struct tcmsg *t;
2767         u32 parent;
2768         u32 chain_index;
2769         struct Qdisc *q;
2770         struct tcf_chain *chain;
2771         struct tcf_block *block;
2772         unsigned long cl;
2773         int err;
2774
2775         if (n->nlmsg_type != RTM_GETCHAIN &&
2776             !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2777                 return -EPERM;
2778
2779 replay:
2780         q = NULL;
2781         err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2782                                      rtm_tca_policy, extack);
2783         if (err < 0)
2784                 return err;
2785
2786         t = nlmsg_data(n);
2787         parent = t->tcm_parent;
2788         cl = 0;
2789
2790         block = tcf_block_find(net, &q, &parent, &cl,
2791                                t->tcm_ifindex, t->tcm_block_index, extack);
2792         if (IS_ERR(block))
2793                 return PTR_ERR(block);
2794
2795         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2796         if (chain_index > TC_ACT_EXT_VAL_MASK) {
2797                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2798                 err = -EINVAL;
2799                 goto errout_block;
2800         }
2801
2802         mutex_lock(&block->lock);
2803         chain = tcf_chain_lookup(block, chain_index);
2804         if (n->nlmsg_type == RTM_NEWCHAIN) {
2805                 if (chain) {
2806                         if (tcf_chain_held_by_acts_only(chain)) {
2807                                 /* The chain exists only because there is
2808                                  * some action referencing it.
2809                                  */
2810                                 tcf_chain_hold(chain);
2811                         } else {
2812                                 NL_SET_ERR_MSG(extack, "Filter chain already exists");
2813                                 err = -EEXIST;
2814                                 goto errout_block_locked;
2815                         }
2816                 } else {
2817                         if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2818                                 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
2819                                 err = -ENOENT;
2820                                 goto errout_block_locked;
2821                         }
2822                         chain = tcf_chain_create(block, chain_index);
2823                         if (!chain) {
2824                                 NL_SET_ERR_MSG(extack, "Failed to create filter chain");
2825                                 err = -ENOMEM;
2826                                 goto errout_block_locked;
2827                         }
2828                 }
2829         } else {
2830                 if (!chain || tcf_chain_held_by_acts_only(chain)) {
2831                         NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2832                         err = -EINVAL;
2833                         goto errout_block_locked;
2834                 }
2835                 tcf_chain_hold(chain);
2836         }
2837
2838         if (n->nlmsg_type == RTM_NEWCHAIN) {
2839                 /* Modifying chain requires holding parent block lock. In case
2840                  * the chain was successfully added, take a reference to the
2841                  * chain. This ensures that an empty chain does not disappear at
2842                  * the end of this function.
2843                  */
2844                 tcf_chain_hold(chain);
2845                 chain->explicitly_created = true;
2846         }
2847         mutex_unlock(&block->lock);
2848
2849         switch (n->nlmsg_type) {
2850         case RTM_NEWCHAIN:
2851                 err = tc_chain_tmplt_add(chain, net, tca, extack);
2852                 if (err) {
2853                         tcf_chain_put_explicitly_created(chain);
2854                         goto errout;
2855                 }
2856
2857                 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2858                                 RTM_NEWCHAIN, false);
2859                 break;
2860         case RTM_DELCHAIN:
2861                 tfilter_notify_chain(net, skb, block, q, parent, n,
2862                                      chain, RTM_DELTFILTER, true);
2863                 /* Flush the chain first as the user requested chain removal. */
2864                 tcf_chain_flush(chain, true);
2865                 /* In case the chain was successfully deleted, put a reference
2866                  * to the chain previously taken during addition.
2867                  */
2868                 tcf_chain_put_explicitly_created(chain);
2869                 break;
2870         case RTM_GETCHAIN:
2871                 err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2872                                       n->nlmsg_flags, n->nlmsg_type, true);
2873                 if (err < 0)
2874                         NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2875                 break;
2876         default:
2877                 err = -EOPNOTSUPP;
2878                 NL_SET_ERR_MSG(extack, "Unsupported message type");
2879                 goto errout;
2880         }
2881
2882 errout:
2883         tcf_chain_put(chain);
2884 errout_block:
2885         tcf_block_release(q, block, true);
2886         if (err == -EAGAIN)
2887                 /* Replay the request. */
2888                 goto replay;
2889         return err;
2890
2891 errout_block_locked:
2892         mutex_unlock(&block->lock);
2893         goto errout_block;
2894 }
2895
2896 /* called with RTNL */
2897 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2898 {
2899         struct net *net = sock_net(skb->sk);
2900         struct nlattr *tca[TCA_MAX + 1];
2901         struct Qdisc *q = NULL;
2902         struct tcf_block *block;
2903         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2904         struct tcf_chain *chain;
2905         long index_start;
2906         long index;
2907         u32 parent;
2908         int err;
2909
2910         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2911                 return skb->len;
2912
2913         err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2914                                      rtm_tca_policy, cb->extack);
2915         if (err)
2916                 return err;
2917
2918         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2919                 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2920                 if (!block)
2921                         goto out;
2922                 /* If we work with block index, q is NULL and parent value
2923                  * will never be used in the following code. The check
2924                  * in tcf_fill_node prevents it. However, compiler does not
2925                  * see that far, so set parent to zero to silence the warning
2926                  * about parent being uninitialized.
2927                  */
2928                 parent = 0;
2929         } else {
2930                 const struct Qdisc_class_ops *cops;
2931                 struct net_device *dev;
2932                 unsigned long cl = 0;
2933
2934                 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2935                 if (!dev)
2936                         return skb->len;
2937
2938                 parent = tcm->tcm_parent;
2939                 if (!parent) {
2940                         q = dev->qdisc;
2941                         parent = q->handle;
2942                 } else {
2943                         q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2944                 }
2945                 if (!q)
2946                         goto out;
2947                 cops = q->ops->cl_ops;
2948                 if (!cops)
2949                         goto out;
2950                 if (!cops->tcf_block)
2951                         goto out;
2952                 if (TC_H_MIN(tcm->tcm_parent)) {
2953                         cl = cops->find(q, tcm->tcm_parent);
2954                         if (cl == 0)
2955                                 goto out;
2956                 }
2957                 block = cops->tcf_block(q, cl, NULL);
2958                 if (!block)
2959                         goto out;
2960                 if (tcf_block_shared(block))
2961                         q = NULL;
2962         }
2963
2964         index_start = cb->args[0];
2965         index = 0;
2966
2967         mutex_lock(&block->lock);
2968         list_for_each_entry(chain, &block->chain_list, list) {
2969                 if ((tca[TCA_CHAIN] &&
2970                      nla_get_u32(tca[TCA_CHAIN]) != chain->index))
2971                         continue;
2972                 if (index < index_start) {
2973                         index++;
2974                         continue;
2975                 }
2976                 if (tcf_chain_held_by_acts_only(chain))
2977                         continue;
2978                 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2979                                          chain->index, net, skb, block,
2980                                          NETLINK_CB(cb->skb).portid,
2981                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
2982                                          RTM_NEWCHAIN);
2983                 if (err <= 0)
2984                         break;
2985                 index++;
2986         }
2987         mutex_unlock(&block->lock);
2988
2989         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2990                 tcf_block_refcnt_put(block, true);
2991         cb->args[0] = index;
2992
2993 out:
2994         /* If we did no progress, the error (EMSGSIZE) is real */
2995         if (skb->len == 0 && err)
2996                 return err;
2997         return skb->len;
2998 }
2999
3000 void tcf_exts_destroy(struct tcf_exts *exts)
3001 {
3002 #ifdef CONFIG_NET_CLS_ACT
3003         if (exts->actions) {
3004                 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
3005                 kfree(exts->actions);
3006         }
3007         exts->nr_actions = 0;
3008 #endif
3009 }
3010 EXPORT_SYMBOL(tcf_exts_destroy);
3011
3012 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3013                       struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr,
3014                       bool rtnl_held, struct netlink_ext_ack *extack)
3015 {
3016 #ifdef CONFIG_NET_CLS_ACT
3017         {
3018                 struct tc_action *act;
3019                 size_t attr_size = 0;
3020
3021                 if (exts->police && tb[exts->police]) {
3022                         act = tcf_action_init_1(net, tp, tb[exts->police],
3023                                                 rate_tlv, "police", ovr,
3024                                                 TCA_ACT_BIND, rtnl_held,
3025                                                 extack);
3026                         if (IS_ERR(act))
3027                                 return PTR_ERR(act);
3028
3029                         act->type = exts->type = TCA_OLD_COMPAT;
3030                         exts->actions[0] = act;
3031                         exts->nr_actions = 1;
3032                         tcf_idr_insert_many(exts->actions);
3033                 } else if (exts->action && tb[exts->action]) {
3034                         int err;
3035
3036                         err = tcf_action_init(net, tp, tb[exts->action],
3037                                               rate_tlv, NULL, ovr, TCA_ACT_BIND,
3038                                               exts->actions, &attr_size,
3039                                               rtnl_held, extack);
3040                         if (err < 0)
3041                                 return err;
3042                         exts->nr_actions = err;
3043                 }
3044         }
3045 #else
3046         if ((exts->action && tb[exts->action]) ||
3047             (exts->police && tb[exts->police])) {
3048                 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
3049                 return -EOPNOTSUPP;
3050         }
3051 #endif
3052
3053         return 0;
3054 }
3055 EXPORT_SYMBOL(tcf_exts_validate);
3056
3057 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
3058 {
3059 #ifdef CONFIG_NET_CLS_ACT
3060         struct tcf_exts old = *dst;
3061
3062         *dst = *src;
3063         tcf_exts_destroy(&old);
3064 #endif
3065 }
3066 EXPORT_SYMBOL(tcf_exts_change);
3067
3068 #ifdef CONFIG_NET_CLS_ACT
3069 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
3070 {
3071         if (exts->nr_actions == 0)
3072                 return NULL;
3073         else
3074                 return exts->actions[0];
3075 }
3076 #endif
3077
3078 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
3079 {
3080 #ifdef CONFIG_NET_CLS_ACT
3081         struct nlattr *nest;
3082
3083         if (exts->action && tcf_exts_has_actions(exts)) {
3084                 /*
3085                  * again for backward compatible mode - we want
3086                  * to work with both old and new modes of entering
3087                  * tc data even if iproute2  was newer - jhs
3088                  */
3089                 if (exts->type != TCA_OLD_COMPAT) {
3090                         nest = nla_nest_start_noflag(skb, exts->action);
3091                         if (nest == NULL)
3092                                 goto nla_put_failure;
3093
3094                         if (tcf_action_dump(skb, exts->actions, 0, 0) < 0)
3095                                 goto nla_put_failure;
3096                         nla_nest_end(skb, nest);
3097                 } else if (exts->police) {
3098                         struct tc_action *act = tcf_exts_first_act(exts);
3099                         nest = nla_nest_start_noflag(skb, exts->police);
3100                         if (nest == NULL || !act)
3101                                 goto nla_put_failure;
3102                         if (tcf_action_dump_old(skb, act, 0, 0) < 0)
3103                                 goto nla_put_failure;
3104                         nla_nest_end(skb, nest);
3105                 }
3106         }
3107         return 0;
3108
3109 nla_put_failure:
3110         nla_nest_cancel(skb, nest);
3111         return -1;
3112 #else
3113         return 0;
3114 #endif
3115 }
3116 EXPORT_SYMBOL(tcf_exts_dump);
3117
3118
3119 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
3120 {
3121 #ifdef CONFIG_NET_CLS_ACT
3122         struct tc_action *a = tcf_exts_first_act(exts);
3123         if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
3124                 return -1;
3125 #endif
3126         return 0;
3127 }
3128 EXPORT_SYMBOL(tcf_exts_dump_stats);
3129
3130 static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
3131 {
3132         if (*flags & TCA_CLS_FLAGS_IN_HW)
3133                 return;
3134         *flags |= TCA_CLS_FLAGS_IN_HW;
3135         atomic_inc(&block->offloadcnt);
3136 }
3137
3138 static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
3139 {
3140         if (!(*flags & TCA_CLS_FLAGS_IN_HW))
3141                 return;
3142         *flags &= ~TCA_CLS_FLAGS_IN_HW;
3143         atomic_dec(&block->offloadcnt);
3144 }
3145
3146 static void tc_cls_offload_cnt_update(struct tcf_block *block,
3147                                       struct tcf_proto *tp, u32 *cnt,
3148                                       u32 *flags, u32 diff, bool add)
3149 {
3150         lockdep_assert_held(&block->cb_lock);
3151
3152         spin_lock(&tp->lock);
3153         if (add) {
3154                 if (!*cnt)
3155                         tcf_block_offload_inc(block, flags);
3156                 *cnt += diff;
3157         } else {
3158                 *cnt -= diff;
3159                 if (!*cnt)
3160                         tcf_block_offload_dec(block, flags);
3161         }
3162         spin_unlock(&tp->lock);
3163 }
3164
3165 static void
3166 tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
3167                          u32 *cnt, u32 *flags)
3168 {
3169         lockdep_assert_held(&block->cb_lock);
3170
3171         spin_lock(&tp->lock);
3172         tcf_block_offload_dec(block, flags);
3173         *cnt = 0;
3174         spin_unlock(&tp->lock);
3175 }
3176
3177 static int
3178 __tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3179                    void *type_data, bool err_stop)
3180 {
3181         struct flow_block_cb *block_cb;
3182         int ok_count = 0;
3183         int err;
3184
3185         list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
3186                 err = block_cb->cb(type, type_data, block_cb->cb_priv);
3187                 if (err) {
3188                         if (err_stop)
3189                                 return err;
3190                 } else {
3191                         ok_count++;
3192                 }
3193         }
3194         return ok_count;
3195 }
3196
3197 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3198                      void *type_data, bool err_stop, bool rtnl_held)
3199 {
3200         bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3201         int ok_count;
3202
3203 retry:
3204         if (take_rtnl)
3205                 rtnl_lock();
3206         down_read(&block->cb_lock);
3207         /* Need to obtain rtnl lock if block is bound to devs that require it.
3208          * In block bind code cb_lock is obtained while holding rtnl, so we must
3209          * obtain the locks in same order here.
3210          */
3211         if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3212                 up_read(&block->cb_lock);
3213                 take_rtnl = true;
3214                 goto retry;
3215         }
3216
3217         ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3218
3219         up_read(&block->cb_lock);
3220         if (take_rtnl)
3221                 rtnl_unlock();
3222         return ok_count;
3223 }
3224 EXPORT_SYMBOL(tc_setup_cb_call);
3225
3226 /* Non-destructive filter add. If filter that wasn't already in hardware is
3227  * successfully offloaded, increment block offloads counter. On failure,
3228  * previously offloaded filter is considered to be intact and offloads counter
3229  * is not decremented.
3230  */
3231
3232 int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
3233                     enum tc_setup_type type, void *type_data, bool err_stop,
3234                     u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3235 {
3236         bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3237         int ok_count;
3238
3239 retry:
3240         if (take_rtnl)
3241                 rtnl_lock();
3242         down_read(&block->cb_lock);
3243         /* Need to obtain rtnl lock if block is bound to devs that require it.
3244          * In block bind code cb_lock is obtained while holding rtnl, so we must
3245          * obtain the locks in same order here.
3246          */
3247         if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3248                 up_read(&block->cb_lock);
3249                 take_rtnl = true;
3250                 goto retry;
3251         }
3252
3253         /* Make sure all netdevs sharing this block are offload-capable. */
3254         if (block->nooffloaddevcnt && err_stop) {
3255                 ok_count = -EOPNOTSUPP;
3256                 goto err_unlock;
3257         }
3258
3259         ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3260         if (ok_count < 0)
3261                 goto err_unlock;
3262
3263         if (tp->ops->hw_add)
3264                 tp->ops->hw_add(tp, type_data);
3265         if (ok_count > 0)
3266                 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags,
3267                                           ok_count, true);
3268 err_unlock:
3269         up_read(&block->cb_lock);
3270         if (take_rtnl)
3271                 rtnl_unlock();
3272         return ok_count < 0 ? ok_count : 0;
3273 }
3274 EXPORT_SYMBOL(tc_setup_cb_add);
3275
3276 /* Destructive filter replace. If filter that wasn't already in hardware is
3277  * successfully offloaded, increment block offload counter. On failure,
3278  * previously offloaded filter is considered to be destroyed and offload counter
3279  * is decremented.
3280  */
3281
3282 int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
3283                         enum tc_setup_type type, void *type_data, bool err_stop,
3284                         u32 *old_flags, unsigned int *old_in_hw_count,
3285                         u32 *new_flags, unsigned int *new_in_hw_count,
3286                         bool rtnl_held)
3287 {
3288         bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3289         int ok_count;
3290
3291 retry:
3292         if (take_rtnl)
3293                 rtnl_lock();
3294         down_read(&block->cb_lock);
3295         /* Need to obtain rtnl lock if block is bound to devs that require it.
3296          * In block bind code cb_lock is obtained while holding rtnl, so we must
3297          * obtain the locks in same order here.
3298          */
3299         if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3300                 up_read(&block->cb_lock);
3301                 take_rtnl = true;
3302                 goto retry;
3303         }
3304
3305         /* Make sure all netdevs sharing this block are offload-capable. */
3306         if (block->nooffloaddevcnt && err_stop) {
3307                 ok_count = -EOPNOTSUPP;
3308                 goto err_unlock;
3309         }
3310
3311         tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags);
3312         if (tp->ops->hw_del)
3313                 tp->ops->hw_del(tp, type_data);
3314
3315         ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3316         if (ok_count < 0)
3317                 goto err_unlock;
3318
3319         if (tp->ops->hw_add)
3320                 tp->ops->hw_add(tp, type_data);
3321         if (ok_count > 0)
3322                 tc_cls_offload_cnt_update(block, tp, new_in_hw_count,
3323                                           new_flags, ok_count, true);
3324 err_unlock:
3325         up_read(&block->cb_lock);
3326         if (take_rtnl)
3327                 rtnl_unlock();
3328         return ok_count < 0 ? ok_count : 0;
3329 }
3330 EXPORT_SYMBOL(tc_setup_cb_replace);
3331
3332 /* Destroy filter and decrement block offload counter, if filter was previously
3333  * offloaded.
3334  */
3335
3336 int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
3337                         enum tc_setup_type type, void *type_data, bool err_stop,
3338                         u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3339 {
3340         bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3341         int ok_count;
3342
3343 retry:
3344         if (take_rtnl)
3345                 rtnl_lock();
3346         down_read(&block->cb_lock);
3347         /* Need to obtain rtnl lock if block is bound to devs that require it.
3348          * In block bind code cb_lock is obtained while holding rtnl, so we must
3349          * obtain the locks in same order here.
3350          */
3351         if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3352                 up_read(&block->cb_lock);
3353                 take_rtnl = true;
3354                 goto retry;
3355         }
3356
3357         ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3358
3359         tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags);
3360         if (tp->ops->hw_del)
3361                 tp->ops->hw_del(tp, type_data);
3362
3363         up_read(&block->cb_lock);
3364         if (take_rtnl)
3365                 rtnl_unlock();
3366         return ok_count < 0 ? ok_count : 0;
3367 }
3368 EXPORT_SYMBOL(tc_setup_cb_destroy);
3369
3370 int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
3371                           bool add, flow_setup_cb_t *cb,
3372                           enum tc_setup_type type, void *type_data,
3373                           void *cb_priv, u32 *flags, unsigned int *in_hw_count)
3374 {
3375         int err = cb(type, type_data, cb_priv);
3376
3377         if (err) {
3378                 if (add && tc_skip_sw(*flags))
3379                         return err;
3380         } else {
3381                 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1,
3382                                           add);
3383         }
3384
3385         return 0;
3386 }
3387 EXPORT_SYMBOL(tc_setup_cb_reoffload);
3388
3389 void tc_cleanup_flow_action(struct flow_action *flow_action)
3390 {
3391         struct flow_action_entry *entry;
3392         int i;
3393
3394         flow_action_for_each(i, entry, flow_action)
3395                 if (entry->destructor)
3396                         entry->destructor(entry->destructor_priv);
3397 }
3398 EXPORT_SYMBOL(tc_cleanup_flow_action);
3399
3400 static void tcf_mirred_get_dev(struct flow_action_entry *entry,
3401                                const struct tc_action *act)
3402 {
3403 #ifdef CONFIG_NET_CLS_ACT
3404         entry->dev = act->ops->get_dev(act, &entry->destructor);
3405         if (!entry->dev)
3406                 return;
3407         entry->destructor_priv = entry->dev;
3408 #endif
3409 }
3410
3411 static void tcf_tunnel_encap_put_tunnel(void *priv)
3412 {
3413         struct ip_tunnel_info *tunnel = priv;
3414
3415         kfree(tunnel);
3416 }
3417
3418 static int tcf_tunnel_encap_get_tunnel(struct flow_action_entry *entry,
3419                                        const struct tc_action *act)
3420 {
3421         entry->tunnel = tcf_tunnel_info_copy(act);
3422         if (!entry->tunnel)
3423                 return -ENOMEM;
3424         entry->destructor = tcf_tunnel_encap_put_tunnel;
3425         entry->destructor_priv = entry->tunnel;
3426         return 0;
3427 }
3428
3429 static void tcf_sample_get_group(struct flow_action_entry *entry,
3430                                  const struct tc_action *act)
3431 {
3432 #ifdef CONFIG_NET_CLS_ACT
3433         entry->sample.psample_group =
3434                 act->ops->get_psample_group(act, &entry->destructor);
3435         entry->destructor_priv = entry->sample.psample_group;
3436 #endif
3437 }
3438
3439 int tc_setup_flow_action(struct flow_action *flow_action,
3440                          const struct tcf_exts *exts, bool rtnl_held)
3441 {
3442         struct tc_action *act;
3443         int i, j, k, err = 0;
3444
3445         if (!exts)
3446                 return 0;
3447
3448         if (!rtnl_held)
3449                 rtnl_lock();
3450
3451         j = 0;
3452         tcf_exts_for_each_action(i, act, exts) {
3453                 struct flow_action_entry *entry;
3454
3455                 entry = &flow_action->entries[j];
3456                 spin_lock_bh(&act->tcfa_lock);
3457                 if (is_tcf_gact_ok(act)) {
3458                         entry->id = FLOW_ACTION_ACCEPT;
3459                 } else if (is_tcf_gact_shot(act)) {
3460                         entry->id = FLOW_ACTION_DROP;
3461                 } else if (is_tcf_gact_trap(act)) {
3462                         entry->id = FLOW_ACTION_TRAP;
3463                 } else if (is_tcf_gact_goto_chain(act)) {
3464                         entry->id = FLOW_ACTION_GOTO;
3465                         entry->chain_index = tcf_gact_goto_chain_index(act);
3466                 } else if (is_tcf_mirred_egress_redirect(act)) {
3467                         entry->id = FLOW_ACTION_REDIRECT;
3468                         tcf_mirred_get_dev(entry, act);
3469                 } else if (is_tcf_mirred_egress_mirror(act)) {
3470                         entry->id = FLOW_ACTION_MIRRED;
3471                         tcf_mirred_get_dev(entry, act);
3472                 } else if (is_tcf_mirred_ingress_redirect(act)) {
3473                         entry->id = FLOW_ACTION_REDIRECT_INGRESS;
3474                         tcf_mirred_get_dev(entry, act);
3475                 } else if (is_tcf_mirred_ingress_mirror(act)) {
3476                         entry->id = FLOW_ACTION_MIRRED_INGRESS;
3477                         tcf_mirred_get_dev(entry, act);
3478                 } else if (is_tcf_vlan(act)) {
3479                         switch (tcf_vlan_action(act)) {
3480                         case TCA_VLAN_ACT_PUSH:
3481                                 entry->id = FLOW_ACTION_VLAN_PUSH;
3482                                 entry->vlan.vid = tcf_vlan_push_vid(act);
3483                                 entry->vlan.proto = tcf_vlan_push_proto(act);
3484                                 entry->vlan.prio = tcf_vlan_push_prio(act);
3485                                 break;
3486                         case TCA_VLAN_ACT_POP:
3487                                 entry->id = FLOW_ACTION_VLAN_POP;
3488                                 break;
3489                         case TCA_VLAN_ACT_MODIFY:
3490                                 entry->id = FLOW_ACTION_VLAN_MANGLE;
3491                                 entry->vlan.vid = tcf_vlan_push_vid(act);
3492                                 entry->vlan.proto = tcf_vlan_push_proto(act);
3493                                 entry->vlan.prio = tcf_vlan_push_prio(act);
3494                                 break;
3495                         default:
3496                                 err = -EOPNOTSUPP;
3497                                 goto err_out_locked;
3498                         }
3499                 } else if (is_tcf_tunnel_set(act)) {
3500                         entry->id = FLOW_ACTION_TUNNEL_ENCAP;
3501                         err = tcf_tunnel_encap_get_tunnel(entry, act);
3502                         if (err)
3503                                 goto err_out_locked;
3504                 } else if (is_tcf_tunnel_release(act)) {
3505                         entry->id = FLOW_ACTION_TUNNEL_DECAP;
3506                 } else if (is_tcf_pedit(act)) {
3507                         for (k = 0; k < tcf_pedit_nkeys(act); k++) {
3508                                 switch (tcf_pedit_cmd(act, k)) {
3509                                 case TCA_PEDIT_KEY_EX_CMD_SET:
3510                                         entry->id = FLOW_ACTION_MANGLE;
3511                                         break;
3512                                 case TCA_PEDIT_KEY_EX_CMD_ADD:
3513                                         entry->id = FLOW_ACTION_ADD;
3514                                         break;
3515                                 default:
3516                                         err = -EOPNOTSUPP;
3517                                         goto err_out_locked;
3518                                 }
3519                                 entry->mangle.htype = tcf_pedit_htype(act, k);
3520                                 entry->mangle.mask = tcf_pedit_mask(act, k);
3521                                 entry->mangle.val = tcf_pedit_val(act, k);
3522                                 entry->mangle.offset = tcf_pedit_offset(act, k);
3523                                 entry = &flow_action->entries[++j];
3524                         }
3525                 } else if (is_tcf_csum(act)) {
3526                         entry->id = FLOW_ACTION_CSUM;
3527                         entry->csum_flags = tcf_csum_update_flags(act);
3528                 } else if (is_tcf_skbedit_mark(act)) {
3529                         entry->id = FLOW_ACTION_MARK;
3530                         entry->mark = tcf_skbedit_mark(act);
3531                 } else if (is_tcf_sample(act)) {
3532                         entry->id = FLOW_ACTION_SAMPLE;
3533                         entry->sample.trunc_size = tcf_sample_trunc_size(act);
3534                         entry->sample.truncate = tcf_sample_truncate(act);
3535                         entry->sample.rate = tcf_sample_rate(act);
3536                         tcf_sample_get_group(entry, act);
3537                 } else if (is_tcf_police(act)) {
3538                         entry->id = FLOW_ACTION_POLICE;
3539                         entry->police.burst = tcf_police_tcfp_burst(act);
3540                         entry->police.rate_bytes_ps =
3541                                 tcf_police_rate_bytes_ps(act);
3542                 } else if (is_tcf_ct(act)) {
3543                         entry->id = FLOW_ACTION_CT;
3544                         entry->ct.action = tcf_ct_action(act);
3545                         entry->ct.zone = tcf_ct_zone(act);
3546                 } else if (is_tcf_mpls(act)) {
3547                         switch (tcf_mpls_action(act)) {
3548                         case TCA_MPLS_ACT_PUSH:
3549                                 entry->id = FLOW_ACTION_MPLS_PUSH;
3550                                 entry->mpls_push.proto = tcf_mpls_proto(act);
3551                                 entry->mpls_push.label = tcf_mpls_label(act);
3552                                 entry->mpls_push.tc = tcf_mpls_tc(act);
3553                                 entry->mpls_push.bos = tcf_mpls_bos(act);
3554                                 entry->mpls_push.ttl = tcf_mpls_ttl(act);
3555                                 break;
3556                         case TCA_MPLS_ACT_POP:
3557                                 entry->id = FLOW_ACTION_MPLS_POP;
3558                                 entry->mpls_pop.proto = tcf_mpls_proto(act);
3559                                 break;
3560                         case TCA_MPLS_ACT_MODIFY:
3561                                 entry->id = FLOW_ACTION_MPLS_MANGLE;
3562                                 entry->mpls_mangle.label = tcf_mpls_label(act);
3563                                 entry->mpls_mangle.tc = tcf_mpls_tc(act);
3564                                 entry->mpls_mangle.bos = tcf_mpls_bos(act);
3565                                 entry->mpls_mangle.ttl = tcf_mpls_ttl(act);
3566                                 break;
3567                         default:
3568                                 err = -EOPNOTSUPP;
3569                                 goto err_out_locked;
3570                         }
3571                 } else if (is_tcf_skbedit_ptype(act)) {
3572                         entry->id = FLOW_ACTION_PTYPE;
3573                         entry->ptype = tcf_skbedit_ptype(act);
3574                 } else {
3575                         err = -EOPNOTSUPP;
3576                         goto err_out_locked;
3577                 }
3578                 spin_unlock_bh(&act->tcfa_lock);
3579
3580                 if (!is_tcf_pedit(act))
3581                         j++;
3582         }
3583
3584 err_out:
3585         if (!rtnl_held)
3586                 rtnl_unlock();
3587
3588         if (err)
3589                 tc_cleanup_flow_action(flow_action);
3590
3591         return err;
3592 err_out_locked:
3593         spin_unlock_bh(&act->tcfa_lock);
3594         goto err_out;
3595 }
3596 EXPORT_SYMBOL(tc_setup_flow_action);
3597
3598 unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
3599 {
3600         unsigned int num_acts = 0;
3601         struct tc_action *act;
3602         int i;
3603
3604         tcf_exts_for_each_action(i, act, exts) {
3605                 if (is_tcf_pedit(act))
3606                         num_acts += tcf_pedit_nkeys(act);
3607                 else
3608                         num_acts++;
3609         }
3610         return num_acts;
3611 }
3612 EXPORT_SYMBOL(tcf_exts_num_actions);
3613
3614 static __net_init int tcf_net_init(struct net *net)
3615 {
3616         struct tcf_net *tn = net_generic(net, tcf_net_id);
3617
3618         spin_lock_init(&tn->idr_lock);
3619         idr_init(&tn->idr);
3620         return 0;
3621 }
3622
3623 static void __net_exit tcf_net_exit(struct net *net)
3624 {
3625         struct tcf_net *tn = net_generic(net, tcf_net_id);
3626
3627         idr_destroy(&tn->idr);
3628 }
3629
3630 static struct pernet_operations tcf_net_ops = {
3631         .init = tcf_net_init,
3632         .exit = tcf_net_exit,
3633         .id   = &tcf_net_id,
3634         .size = sizeof(struct tcf_net),
3635 };
3636
3637 static struct flow_indr_block_entry block_entry = {
3638         .cb = tc_indr_block_get_and_cmd,
3639         .list = LIST_HEAD_INIT(block_entry.list),
3640 };
3641
3642 static int __init tc_filter_init(void)
3643 {
3644         int err;
3645
3646         tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
3647         if (!tc_filter_wq)
3648                 return -ENOMEM;
3649
3650         err = register_pernet_subsys(&tcf_net_ops);
3651         if (err)
3652                 goto err_register_pernet_subsys;
3653
3654         flow_indr_add_block_cb(&block_entry);
3655
3656         rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
3657                       RTNL_FLAG_DOIT_UNLOCKED);
3658         rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
3659                       RTNL_FLAG_DOIT_UNLOCKED);
3660         rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
3661                       tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED);
3662         rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
3663         rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
3664         rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
3665                       tc_dump_chain, 0);
3666
3667         return 0;
3668
3669 err_register_pernet_subsys:
3670         destroy_workqueue(tc_filter_wq);
3671         return err;
3672 }
3673
3674 subsys_initcall(tc_filter_init);