1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * net/sched/sch_api.c Packet scheduler API.
5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
9 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
10 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
11 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
14 #include <linux/module.h>
15 #include <linux/types.h>
16 #include <linux/kernel.h>
17 #include <linux/string.h>
18 #include <linux/errno.h>
19 #include <linux/skbuff.h>
20 #include <linux/init.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/kmod.h>
24 #include <linux/list.h>
25 #include <linux/hrtimer.h>
26 #include <linux/slab.h>
27 #include <linux/hashtable.h>
29 #include <net/net_namespace.h>
31 #include <net/netlink.h>
32 #include <net/pkt_sched.h>
33 #include <net/pkt_cls.h>
35 #include <trace/events/qdisc.h>
42 This file consists of two interrelated parts:
44 1. queueing disciplines manager frontend.
45 2. traffic classes manager frontend.
47 Generally, queueing discipline ("qdisc") is a black box,
48 which is able to enqueue packets and to dequeue them (when
49 device is ready to send something) in order and at times
50 determined by algorithm hidden in it.
52 qdisc's are divided to two categories:
53 - "queues", which have no internal structure visible from outside.
54 - "schedulers", which split all the packets to "traffic classes",
55 using "packet classifiers" (look at cls_api.c)
57 In turn, classes may have child qdiscs (as rule, queues)
58 attached to them etc. etc. etc.
60 The goal of the routines in this file is to translate
61 information supplied by user in the form of handles
62 to more intelligible for kernel form, to make some sanity
63 checks and part of work, which is common to all qdiscs
64 and to provide rtnetlink notifications.
66 All real intelligent work is done inside qdisc modules.
70 Every discipline has two major routines: enqueue and dequeue.
74 dequeue usually returns a skb to send. It is allowed to return NULL,
75 but it does not mean that queue is empty, it just means that
76 discipline does not want to send anything this time.
77 Queue is really empty if q->q.qlen == 0.
78 For complicated disciplines with multiple queues q->q is not
79 real packet queue, but however q->q.qlen must be valid.
83 enqueue returns 0, if packet was enqueued successfully.
84 If packet (this one or another one) was dropped, it returns
86 NET_XMIT_DROP - this packet dropped
87 Expected action: do not backoff, but wait until queue will clear.
88 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
89 Expected action: backoff or ignore
95 like dequeue but without removing a packet from the queue
99 returns qdisc to initial state: purge all buffers, clear all
100 timers, counters (except for statistics) etc.
104 initializes newly created qdisc.
108 destroys resources allocated by init and during lifetime of qdisc.
112 changes qdisc parameters.
115 /* Protects list of registered TC modules. It is pure SMP lock. */
116 static DEFINE_RWLOCK(qdisc_mod_lock);
119 /************************************************
120 * Queueing disciplines manipulation. *
121 ************************************************/
124 /* The list of all installed queueing disciplines. */
126 static struct Qdisc_ops *qdisc_base;
128 /* Register/unregister queueing discipline */
130 int register_qdisc(struct Qdisc_ops *qops)
132 struct Qdisc_ops *q, **qp;
135 write_lock(&qdisc_mod_lock);
136 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
137 if (!strcmp(qops->id, q->id))
140 if (qops->enqueue == NULL)
141 qops->enqueue = noop_qdisc_ops.enqueue;
142 if (qops->peek == NULL) {
143 if (qops->dequeue == NULL)
144 qops->peek = noop_qdisc_ops.peek;
148 if (qops->dequeue == NULL)
149 qops->dequeue = noop_qdisc_ops.dequeue;
152 const struct Qdisc_class_ops *cops = qops->cl_ops;
154 if (!(cops->find && cops->walk && cops->leaf))
157 if (cops->tcf_block && !(cops->bind_tcf && cops->unbind_tcf))
165 write_unlock(&qdisc_mod_lock);
172 EXPORT_SYMBOL(register_qdisc);
174 int unregister_qdisc(struct Qdisc_ops *qops)
176 struct Qdisc_ops *q, **qp;
179 write_lock(&qdisc_mod_lock);
180 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
188 write_unlock(&qdisc_mod_lock);
191 EXPORT_SYMBOL(unregister_qdisc);
193 /* Get default qdisc if not otherwise specified */
194 void qdisc_get_default(char *name, size_t len)
196 read_lock(&qdisc_mod_lock);
197 strlcpy(name, default_qdisc_ops->id, len);
198 read_unlock(&qdisc_mod_lock);
201 static struct Qdisc_ops *qdisc_lookup_default(const char *name)
203 struct Qdisc_ops *q = NULL;
205 for (q = qdisc_base; q; q = q->next) {
206 if (!strcmp(name, q->id)) {
207 if (!try_module_get(q->owner))
216 /* Set new default qdisc to use */
217 int qdisc_set_default(const char *name)
219 const struct Qdisc_ops *ops;
221 if (!capable(CAP_NET_ADMIN))
224 write_lock(&qdisc_mod_lock);
225 ops = qdisc_lookup_default(name);
227 /* Not found, drop lock and try to load module */
228 write_unlock(&qdisc_mod_lock);
229 request_module("sch_%s", name);
230 write_lock(&qdisc_mod_lock);
232 ops = qdisc_lookup_default(name);
236 /* Set new default */
237 module_put(default_qdisc_ops->owner);
238 default_qdisc_ops = ops;
240 write_unlock(&qdisc_mod_lock);
242 return ops ? 0 : -ENOENT;
245 #ifdef CONFIG_NET_SCH_DEFAULT
246 /* Set default value from kernel config */
247 static int __init sch_default_qdisc(void)
249 return qdisc_set_default(CONFIG_DEFAULT_NET_SCH);
251 late_initcall(sch_default_qdisc);
254 /* We know handle. Find qdisc among all qdisc's attached to device
255 * (root qdisc, all its children, children of children etc.)
256 * Note: caller either uses rtnl or rcu_read_lock()
259 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
263 if (!qdisc_dev(root))
264 return (root->handle == handle ? root : NULL);
266 if (!(root->flags & TCQ_F_BUILTIN) &&
267 root->handle == handle)
270 hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle,
271 lockdep_rtnl_is_held()) {
272 if (q->handle == handle)
278 void qdisc_hash_add(struct Qdisc *q, bool invisible)
280 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
282 hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);
284 q->flags |= TCQ_F_INVISIBLE;
287 EXPORT_SYMBOL(qdisc_hash_add);
289 void qdisc_hash_del(struct Qdisc *q)
291 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
293 hash_del_rcu(&q->hash);
296 EXPORT_SYMBOL(qdisc_hash_del);
298 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
304 q = qdisc_match_from_root(rtnl_dereference(dev->qdisc), handle);
308 if (dev_ingress_queue(dev))
309 q = qdisc_match_from_root(
310 dev_ingress_queue(dev)->qdisc_sleeping,
316 struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle)
318 struct netdev_queue *nq;
323 q = qdisc_match_from_root(rcu_dereference(dev->qdisc), handle);
327 nq = dev_ingress_queue_rcu(dev);
329 q = qdisc_match_from_root(nq->qdisc_sleeping, handle);
334 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
337 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
341 cl = cops->find(p, classid);
345 return cops->leaf(p, cl);
348 /* Find queueing discipline by name */
350 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
352 struct Qdisc_ops *q = NULL;
355 read_lock(&qdisc_mod_lock);
356 for (q = qdisc_base; q; q = q->next) {
357 if (nla_strcmp(kind, q->id) == 0) {
358 if (!try_module_get(q->owner))
363 read_unlock(&qdisc_mod_lock);
368 /* The linklayer setting were not transferred from iproute2, in older
369 * versions, and the rate tables lookup systems have been dropped in
370 * the kernel. To keep backward compatible with older iproute2 tc
371 * utils, we detect the linklayer setting by detecting if the rate
372 * table were modified.
374 * For linklayer ATM table entries, the rate table will be aligned to
375 * 48 bytes, thus some table entries will contain the same value. The
376 * mpu (min packet unit) is also encoded into the old rate table, thus
377 * starting from the mpu, we find low and high table entries for
378 * mapping this cell. If these entries contain the same value, when
379 * the rate tables have been modified for linklayer ATM.
381 * This is done by rounding mpu to the nearest 48 bytes cell/entry,
382 * and then roundup to the next cell, calc the table entry one below,
385 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab)
387 int low = roundup(r->mpu, 48);
388 int high = roundup(low+1, 48);
389 int cell_low = low >> r->cell_log;
390 int cell_high = (high >> r->cell_log) - 1;
392 /* rtab is too inaccurate at rates > 100Mbit/s */
393 if ((r->rate > (100000000/8)) || (rtab[0] == 0)) {
394 pr_debug("TC linklayer: Giving up ATM detection\n");
395 return TC_LINKLAYER_ETHERNET;
398 if ((cell_high > cell_low) && (cell_high < 256)
399 && (rtab[cell_low] == rtab[cell_high])) {
400 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n",
401 cell_low, cell_high, rtab[cell_high]);
402 return TC_LINKLAYER_ATM;
404 return TC_LINKLAYER_ETHERNET;
407 static struct qdisc_rate_table *qdisc_rtab_list;
409 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,
411 struct netlink_ext_ack *extack)
413 struct qdisc_rate_table *rtab;
415 if (tab == NULL || r->rate == 0 ||
416 r->cell_log == 0 || r->cell_log >= 32 ||
417 nla_len(tab) != TC_RTAB_SIZE) {
418 NL_SET_ERR_MSG(extack, "Invalid rate table parameters for searching");
422 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
423 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) &&
424 !memcmp(&rtab->data, nla_data(tab), 1024)) {
430 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
434 memcpy(rtab->data, nla_data(tab), 1024);
435 if (r->linklayer == TC_LINKLAYER_UNAWARE)
436 r->linklayer = __detect_linklayer(r, rtab->data);
437 rtab->next = qdisc_rtab_list;
438 qdisc_rtab_list = rtab;
440 NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table");
444 EXPORT_SYMBOL(qdisc_get_rtab);
446 void qdisc_put_rtab(struct qdisc_rate_table *tab)
448 struct qdisc_rate_table *rtab, **rtabp;
450 if (!tab || --tab->refcnt)
453 for (rtabp = &qdisc_rtab_list;
454 (rtab = *rtabp) != NULL;
455 rtabp = &rtab->next) {
463 EXPORT_SYMBOL(qdisc_put_rtab);
465 static LIST_HEAD(qdisc_stab_list);
467 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
468 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
469 [TCA_STAB_DATA] = { .type = NLA_BINARY },
472 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt,
473 struct netlink_ext_ack *extack)
475 struct nlattr *tb[TCA_STAB_MAX + 1];
476 struct qdisc_size_table *stab;
477 struct tc_sizespec *s;
478 unsigned int tsize = 0;
482 err = nla_parse_nested_deprecated(tb, TCA_STAB_MAX, opt, stab_policy,
486 if (!tb[TCA_STAB_BASE]) {
487 NL_SET_ERR_MSG(extack, "Size table base attribute is missing");
488 return ERR_PTR(-EINVAL);
491 s = nla_data(tb[TCA_STAB_BASE]);
494 if (!tb[TCA_STAB_DATA]) {
495 NL_SET_ERR_MSG(extack, "Size table data attribute is missing");
496 return ERR_PTR(-EINVAL);
498 tab = nla_data(tb[TCA_STAB_DATA]);
499 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
502 if (tsize != s->tsize || (!tab && tsize > 0)) {
503 NL_SET_ERR_MSG(extack, "Invalid size of size table");
504 return ERR_PTR(-EINVAL);
507 list_for_each_entry(stab, &qdisc_stab_list, list) {
508 if (memcmp(&stab->szopts, s, sizeof(*s)))
510 if (tsize > 0 && memcmp(stab->data, tab, tsize * sizeof(u16)))
516 if (s->size_log > STAB_SIZE_LOG_MAX ||
517 s->cell_log > STAB_SIZE_LOG_MAX) {
518 NL_SET_ERR_MSG(extack, "Invalid logarithmic size of size table");
519 return ERR_PTR(-EINVAL);
522 stab = kmalloc(sizeof(*stab) + tsize * sizeof(u16), GFP_KERNEL);
524 return ERR_PTR(-ENOMEM);
529 memcpy(stab->data, tab, tsize * sizeof(u16));
531 list_add_tail(&stab->list, &qdisc_stab_list);
536 void qdisc_put_stab(struct qdisc_size_table *tab)
541 if (--tab->refcnt == 0) {
542 list_del(&tab->list);
546 EXPORT_SYMBOL(qdisc_put_stab);
548 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
552 nest = nla_nest_start_noflag(skb, TCA_STAB);
554 goto nla_put_failure;
555 if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts))
556 goto nla_put_failure;
557 nla_nest_end(skb, nest);
565 void __qdisc_calculate_pkt_len(struct sk_buff *skb,
566 const struct qdisc_size_table *stab)
570 pkt_len = skb->len + stab->szopts.overhead;
571 if (unlikely(!stab->szopts.tsize))
574 slot = pkt_len + stab->szopts.cell_align;
575 if (unlikely(slot < 0))
578 slot >>= stab->szopts.cell_log;
579 if (likely(slot < stab->szopts.tsize))
580 pkt_len = stab->data[slot];
582 pkt_len = stab->data[stab->szopts.tsize - 1] *
583 (slot / stab->szopts.tsize) +
584 stab->data[slot % stab->szopts.tsize];
586 pkt_len <<= stab->szopts.size_log;
588 if (unlikely(pkt_len < 1))
590 qdisc_skb_cb(skb)->pkt_len = pkt_len;
592 EXPORT_SYMBOL(__qdisc_calculate_pkt_len);
594 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc)
596 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
597 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n",
598 txt, qdisc->ops->id, qdisc->handle >> 16);
599 qdisc->flags |= TCQ_F_WARN_NONWC;
602 EXPORT_SYMBOL(qdisc_warn_nonwc);
604 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
606 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
610 __netif_schedule(qdisc_root(wd->qdisc));
613 return HRTIMER_NORESTART;
616 void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc,
619 hrtimer_init(&wd->timer, clockid, HRTIMER_MODE_ABS_PINNED);
620 wd->timer.function = qdisc_watchdog;
623 EXPORT_SYMBOL(qdisc_watchdog_init_clockid);
625 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
627 qdisc_watchdog_init_clockid(wd, qdisc, CLOCK_MONOTONIC);
629 EXPORT_SYMBOL(qdisc_watchdog_init);
631 void qdisc_watchdog_schedule_range_ns(struct qdisc_watchdog *wd, u64 expires,
634 if (test_bit(__QDISC_STATE_DEACTIVATED,
635 &qdisc_root_sleeping(wd->qdisc)->state))
638 if (hrtimer_is_queued(&wd->timer)) {
639 /* If timer is already set in [expires, expires + delta_ns],
640 * do not reprogram it.
642 if (wd->last_expires - expires <= delta_ns)
646 wd->last_expires = expires;
647 hrtimer_start_range_ns(&wd->timer,
648 ns_to_ktime(expires),
650 HRTIMER_MODE_ABS_PINNED);
652 EXPORT_SYMBOL(qdisc_watchdog_schedule_range_ns);
654 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
656 hrtimer_cancel(&wd->timer);
658 EXPORT_SYMBOL(qdisc_watchdog_cancel);
660 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
662 struct hlist_head *h;
665 h = kvmalloc_array(n, sizeof(struct hlist_head), GFP_KERNEL);
668 for (i = 0; i < n; i++)
669 INIT_HLIST_HEAD(&h[i]);
674 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
676 struct Qdisc_class_common *cl;
677 struct hlist_node *next;
678 struct hlist_head *nhash, *ohash;
679 unsigned int nsize, nmask, osize;
682 /* Rehash when load factor exceeds 0.75 */
683 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
685 nsize = clhash->hashsize * 2;
687 nhash = qdisc_class_hash_alloc(nsize);
691 ohash = clhash->hash;
692 osize = clhash->hashsize;
695 for (i = 0; i < osize; i++) {
696 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) {
697 h = qdisc_class_hash(cl->classid, nmask);
698 hlist_add_head(&cl->hnode, &nhash[h]);
701 clhash->hash = nhash;
702 clhash->hashsize = nsize;
703 clhash->hashmask = nmask;
704 sch_tree_unlock(sch);
708 EXPORT_SYMBOL(qdisc_class_hash_grow);
710 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
712 unsigned int size = 4;
714 clhash->hash = qdisc_class_hash_alloc(size);
717 clhash->hashsize = size;
718 clhash->hashmask = size - 1;
719 clhash->hashelems = 0;
722 EXPORT_SYMBOL(qdisc_class_hash_init);
724 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
726 kvfree(clhash->hash);
728 EXPORT_SYMBOL(qdisc_class_hash_destroy);
730 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
731 struct Qdisc_class_common *cl)
735 INIT_HLIST_NODE(&cl->hnode);
736 h = qdisc_class_hash(cl->classid, clhash->hashmask);
737 hlist_add_head(&cl->hnode, &clhash->hash[h]);
740 EXPORT_SYMBOL(qdisc_class_hash_insert);
742 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
743 struct Qdisc_class_common *cl)
745 hlist_del(&cl->hnode);
748 EXPORT_SYMBOL(qdisc_class_hash_remove);
750 /* Allocate an unique handle from space managed by kernel
751 * Possible range is [8000-FFFF]:0000 (0x8000 values)
753 static u32 qdisc_alloc_handle(struct net_device *dev)
756 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
759 autohandle += TC_H_MAKE(0x10000U, 0);
760 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
761 autohandle = TC_H_MAKE(0x80000000U, 0);
762 if (!qdisc_lookup(dev, autohandle))
770 void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len)
772 bool qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED;
773 const struct Qdisc_class_ops *cops;
779 if (n == 0 && len == 0)
781 drops = max_t(int, n, 0);
783 while ((parentid = sch->parent)) {
784 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
787 if (sch->flags & TCQ_F_NOPARENT)
789 /* Notify parent qdisc only if child qdisc becomes empty.
791 * If child was empty even before update then backlog
792 * counter is screwed and we skip notification because
793 * parent class is already passive.
795 * If the original child was offloaded then it is allowed
796 * to be seem as empty, so the parent is notified anyway.
798 notify = !sch->q.qlen && !WARN_ON_ONCE(!n &&
799 !qdisc_is_offloaded);
800 /* TODO: perform the search on a per txq basis */
801 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
803 WARN_ON_ONCE(parentid != TC_H_ROOT);
806 cops = sch->ops->cl_ops;
807 if (notify && cops->qlen_notify) {
808 cl = cops->find(sch, parentid);
809 cops->qlen_notify(sch, cl);
812 sch->qstats.backlog -= len;
813 __qdisc_qstats_drop(sch, drops);
817 EXPORT_SYMBOL(qdisc_tree_reduce_backlog);
819 int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type,
822 struct net_device *dev = qdisc_dev(sch);
825 sch->flags &= ~TCQ_F_OFFLOADED;
826 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
829 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
830 if (err == -EOPNOTSUPP)
834 sch->flags |= TCQ_F_OFFLOADED;
838 EXPORT_SYMBOL(qdisc_offload_dump_helper);
840 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
841 struct Qdisc *new, struct Qdisc *old,
842 enum tc_setup_type type, void *type_data,
843 struct netlink_ext_ack *extack)
845 bool any_qdisc_is_offloaded;
848 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
851 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
853 /* Don't report error if the graft is part of destroy operation. */
854 if (!err || !new || new == &noop_qdisc)
857 /* Don't report error if the parent, the old child and the new
858 * one are not offloaded.
860 any_qdisc_is_offloaded = new->flags & TCQ_F_OFFLOADED;
861 any_qdisc_is_offloaded |= sch && sch->flags & TCQ_F_OFFLOADED;
862 any_qdisc_is_offloaded |= old && old->flags & TCQ_F_OFFLOADED;
864 if (any_qdisc_is_offloaded)
865 NL_SET_ERR_MSG(extack, "Offloading graft operation failed.");
867 EXPORT_SYMBOL(qdisc_offload_graft_helper);
869 static void qdisc_offload_graft_root(struct net_device *dev,
870 struct Qdisc *new, struct Qdisc *old,
871 struct netlink_ext_ack *extack)
873 struct tc_root_qopt_offload graft_offload = {
874 .command = TC_ROOT_GRAFT,
875 .handle = new ? new->handle : 0,
876 .ingress = (new && new->flags & TCQ_F_INGRESS) ||
877 (old && old->flags & TCQ_F_INGRESS),
880 qdisc_offload_graft_helper(dev, NULL, new, old,
881 TC_SETUP_ROOT_QDISC, &graft_offload, extack);
884 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
885 u32 portid, u32 seq, u16 flags, int event)
887 struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL;
888 struct gnet_stats_queue __percpu *cpu_qstats = NULL;
890 struct nlmsghdr *nlh;
891 unsigned char *b = skb_tail_pointer(skb);
893 struct qdisc_size_table *stab;
898 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
901 tcm = nlmsg_data(nlh);
902 tcm->tcm_family = AF_UNSPEC;
905 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
906 tcm->tcm_parent = clid;
907 tcm->tcm_handle = q->handle;
908 tcm->tcm_info = refcount_read(&q->refcnt);
909 if (nla_put_string(skb, TCA_KIND, q->ops->id))
910 goto nla_put_failure;
911 if (q->ops->ingress_block_get) {
912 block_index = q->ops->ingress_block_get(q);
914 nla_put_u32(skb, TCA_INGRESS_BLOCK, block_index))
915 goto nla_put_failure;
917 if (q->ops->egress_block_get) {
918 block_index = q->ops->egress_block_get(q);
920 nla_put_u32(skb, TCA_EGRESS_BLOCK, block_index))
921 goto nla_put_failure;
923 if (q->ops->dump && q->ops->dump(q, skb) < 0)
924 goto nla_put_failure;
925 if (nla_put_u8(skb, TCA_HW_OFFLOAD, !!(q->flags & TCQ_F_OFFLOADED)))
926 goto nla_put_failure;
927 qlen = qdisc_qlen_sum(q);
929 stab = rtnl_dereference(q->stab);
930 if (stab && qdisc_dump_stab(skb, stab) < 0)
931 goto nla_put_failure;
933 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
934 NULL, &d, TCA_PAD) < 0)
935 goto nla_put_failure;
937 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
938 goto nla_put_failure;
940 if (qdisc_is_percpu_stats(q)) {
941 cpu_bstats = q->cpu_bstats;
942 cpu_qstats = q->cpu_qstats;
945 if (gnet_stats_copy_basic(qdisc_root_sleeping_running(q),
946 &d, cpu_bstats, &q->bstats) < 0 ||
947 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||
948 gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
949 goto nla_put_failure;
951 if (gnet_stats_finish_copy(&d) < 0)
952 goto nla_put_failure;
954 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
963 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible)
965 if (q->flags & TCQ_F_BUILTIN)
967 if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible)
973 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
974 struct nlmsghdr *n, u32 clid,
975 struct Qdisc *old, struct Qdisc *new)
978 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
980 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
984 if (old && !tc_qdisc_dump_ignore(old, false)) {
985 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
986 0, RTM_DELQDISC) < 0)
989 if (new && !tc_qdisc_dump_ignore(new, false)) {
990 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
991 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
996 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
997 n->nlmsg_flags & NLM_F_ECHO);
1004 static void notify_and_destroy(struct net *net, struct sk_buff *skb,
1005 struct nlmsghdr *n, u32 clid,
1006 struct Qdisc *old, struct Qdisc *new)
1009 qdisc_notify(net, skb, n, clid, old, new);
1015 static void qdisc_clear_nolock(struct Qdisc *sch)
1017 sch->flags &= ~TCQ_F_NOLOCK;
1018 if (!(sch->flags & TCQ_F_CPUSTATS))
1021 free_percpu(sch->cpu_bstats);
1022 free_percpu(sch->cpu_qstats);
1023 sch->cpu_bstats = NULL;
1024 sch->cpu_qstats = NULL;
1025 sch->flags &= ~TCQ_F_CPUSTATS;
1028 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
1031 * When appropriate send a netlink notification using 'skb'
1034 * On success, destroy old qdisc.
1037 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
1038 struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
1039 struct Qdisc *new, struct Qdisc *old,
1040 struct netlink_ext_ack *extack)
1042 struct Qdisc *q = old;
1043 struct net *net = dev_net(dev);
1045 if (parent == NULL) {
1046 unsigned int i, num_q, ingress;
1047 struct netdev_queue *dev_queue;
1050 num_q = dev->num_tx_queues;
1051 if ((q && q->flags & TCQ_F_INGRESS) ||
1052 (new && new->flags & TCQ_F_INGRESS)) {
1054 if (!dev_ingress_queue(dev)) {
1055 NL_SET_ERR_MSG(extack, "Device does not have an ingress queue");
1060 if (dev->flags & IFF_UP)
1061 dev_deactivate(dev);
1063 qdisc_offload_graft_root(dev, new, old, extack);
1065 if (new && new->ops->attach)
1069 for (i = 0; i < num_q; i++) {
1070 dev_queue = netdev_get_tx_queue(dev, i);
1071 old = dev_graft_qdisc(dev_queue, new);
1074 qdisc_refcount_inc(new);
1078 dev_queue = dev_ingress_queue(dev);
1079 old = dev_graft_qdisc(dev_queue, new);
1084 old = rtnl_dereference(dev->qdisc);
1085 if (new && !new->ops->attach)
1086 qdisc_refcount_inc(new);
1087 rcu_assign_pointer(dev->qdisc, new ? : &noop_qdisc);
1089 notify_and_destroy(net, skb, n, classid, old, new);
1091 if (new && new->ops->attach)
1092 new->ops->attach(new);
1094 notify_and_destroy(net, skb, n, classid, old, new);
1097 if (dev->flags & IFF_UP)
1100 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
1104 /* Only support running class lockless if parent is lockless */
1105 if (new && (new->flags & TCQ_F_NOLOCK) && !(parent->flags & TCQ_F_NOLOCK))
1106 qdisc_clear_nolock(new);
1108 if (!cops || !cops->graft)
1111 cl = cops->find(parent, classid);
1113 NL_SET_ERR_MSG(extack, "Specified class not found");
1117 if (new && new->ops == &noqueue_qdisc_ops) {
1118 NL_SET_ERR_MSG(extack, "Cannot assign noqueue to a class");
1122 err = cops->graft(parent, cl, new, &old, extack);
1125 notify_and_destroy(net, skb, n, classid, old, new);
1130 static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca,
1131 struct netlink_ext_ack *extack)
1135 if (tca[TCA_INGRESS_BLOCK]) {
1136 block_index = nla_get_u32(tca[TCA_INGRESS_BLOCK]);
1139 NL_SET_ERR_MSG(extack, "Ingress block index cannot be 0");
1142 if (!sch->ops->ingress_block_set) {
1143 NL_SET_ERR_MSG(extack, "Ingress block sharing is not supported");
1146 sch->ops->ingress_block_set(sch, block_index);
1148 if (tca[TCA_EGRESS_BLOCK]) {
1149 block_index = nla_get_u32(tca[TCA_EGRESS_BLOCK]);
1152 NL_SET_ERR_MSG(extack, "Egress block index cannot be 0");
1155 if (!sch->ops->egress_block_set) {
1156 NL_SET_ERR_MSG(extack, "Egress block sharing is not supported");
1159 sch->ops->egress_block_set(sch, block_index);
1165 Allocate and initialize new qdisc.
1167 Parameters are passed via opt.
1170 static struct Qdisc *qdisc_create(struct net_device *dev,
1171 struct netdev_queue *dev_queue,
1172 struct Qdisc *p, u32 parent, u32 handle,
1173 struct nlattr **tca, int *errp,
1174 struct netlink_ext_ack *extack)
1177 struct nlattr *kind = tca[TCA_KIND];
1179 struct Qdisc_ops *ops;
1180 struct qdisc_size_table *stab;
1182 ops = qdisc_lookup_ops(kind);
1183 #ifdef CONFIG_MODULES
1184 if (ops == NULL && kind != NULL) {
1185 char name[IFNAMSIZ];
1186 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
1187 /* We dropped the RTNL semaphore in order to
1188 * perform the module load. So, even if we
1189 * succeeded in loading the module we have to
1190 * tell the caller to replay the request. We
1191 * indicate this using -EAGAIN.
1192 * We replay the request because the device may
1193 * go away in the mean time.
1196 request_module("sch_%s", name);
1198 ops = qdisc_lookup_ops(kind);
1200 /* We will try again qdisc_lookup_ops,
1201 * so don't keep a reference.
1203 module_put(ops->owner);
1213 NL_SET_ERR_MSG(extack, "Specified qdisc kind is unknown");
1217 sch = qdisc_alloc(dev_queue, ops, extack);
1223 sch->parent = parent;
1225 if (handle == TC_H_INGRESS) {
1226 if (!(sch->flags & TCQ_F_INGRESS)) {
1227 NL_SET_ERR_MSG(extack,
1228 "Specified parent ID is reserved for ingress and clsact Qdiscs");
1232 handle = TC_H_MAKE(TC_H_INGRESS, 0);
1235 handle = qdisc_alloc_handle(dev);
1237 NL_SET_ERR_MSG(extack, "Maximum number of qdisc handles was exceeded");
1242 if (!netif_is_multiqueue(dev))
1243 sch->flags |= TCQ_F_ONETXQUEUE;
1246 sch->handle = handle;
1248 /* This exist to keep backward compatible with a userspace
1249 * loophole, what allowed userspace to get IFF_NO_QUEUE
1250 * facility on older kernels by setting tx_queue_len=0 (prior
1251 * to qdisc init), and then forgot to reinit tx_queue_len
1252 * before again attaching a qdisc.
1254 if ((dev->priv_flags & IFF_NO_QUEUE) && (dev->tx_queue_len == 0)) {
1255 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
1256 netdev_info(dev, "Caught tx_queue_len zero misconfig\n");
1259 err = qdisc_block_indexes_set(sch, tca, extack);
1264 err = ops->init(sch, tca[TCA_OPTIONS], extack);
1269 if (tca[TCA_STAB]) {
1270 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1272 err = PTR_ERR(stab);
1275 rcu_assign_pointer(sch->stab, stab);
1277 if (tca[TCA_RATE]) {
1278 seqcount_t *running;
1281 if (sch->flags & TCQ_F_MQROOT) {
1282 NL_SET_ERR_MSG(extack, "Cannot attach rate estimator to a multi-queue root qdisc");
1286 if (sch->parent != TC_H_ROOT &&
1287 !(sch->flags & TCQ_F_INGRESS) &&
1288 (!p || !(p->flags & TCQ_F_MQROOT)))
1289 running = qdisc_root_sleeping_running(sch);
1291 running = &sch->running;
1293 err = gen_new_estimator(&sch->bstats,
1300 NL_SET_ERR_MSG(extack, "Failed to generate new estimator");
1305 qdisc_hash_add(sch, false);
1306 trace_qdisc_create(ops, dev, parent);
1311 /* ops->init() failed, we call ->destroy() like qdisc_create_dflt() */
1318 module_put(ops->owner);
1325 * Any broken qdiscs that would require a ops->reset() here?
1326 * The qdisc was never in action so it shouldn't be necessary.
1328 qdisc_put_stab(rtnl_dereference(sch->stab));
1334 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca,
1335 struct netlink_ext_ack *extack)
1337 struct qdisc_size_table *ostab, *stab = NULL;
1340 if (tca[TCA_OPTIONS]) {
1341 if (!sch->ops->change) {
1342 NL_SET_ERR_MSG(extack, "Change operation not supported by specified qdisc");
1345 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
1346 NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
1349 err = sch->ops->change(sch, tca[TCA_OPTIONS], extack);
1354 if (tca[TCA_STAB]) {
1355 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1357 return PTR_ERR(stab);
1360 ostab = rtnl_dereference(sch->stab);
1361 rcu_assign_pointer(sch->stab, stab);
1362 qdisc_put_stab(ostab);
1364 if (tca[TCA_RATE]) {
1365 /* NB: ignores errors from replace_estimator
1366 because change can't be undone. */
1367 if (sch->flags & TCQ_F_MQROOT)
1369 gen_replace_estimator(&sch->bstats,
1373 qdisc_root_sleeping_running(sch),
1380 struct check_loop_arg {
1381 struct qdisc_walker w;
1386 static int check_loop_fn(struct Qdisc *q, unsigned long cl,
1387 struct qdisc_walker *w);
1389 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
1391 struct check_loop_arg arg;
1393 if (q->ops->cl_ops == NULL)
1396 arg.w.stop = arg.w.skip = arg.w.count = 0;
1397 arg.w.fn = check_loop_fn;
1400 q->ops->cl_ops->walk(q, &arg.w);
1401 return arg.w.stop ? -ELOOP : 0;
1405 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
1408 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1409 struct check_loop_arg *arg = (struct check_loop_arg *)w;
1411 leaf = cops->leaf(q, cl);
1413 if (leaf == arg->p || arg->depth > 7)
1415 return check_loop(leaf, arg->p, arg->depth + 1);
1420 const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = {
1421 [TCA_KIND] = { .type = NLA_STRING },
1422 [TCA_RATE] = { .type = NLA_BINARY,
1423 .len = sizeof(struct tc_estimator) },
1424 [TCA_STAB] = { .type = NLA_NESTED },
1425 [TCA_DUMP_INVISIBLE] = { .type = NLA_FLAG },
1426 [TCA_CHAIN] = { .type = NLA_U32 },
1427 [TCA_INGRESS_BLOCK] = { .type = NLA_U32 },
1428 [TCA_EGRESS_BLOCK] = { .type = NLA_U32 },
1435 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1436 struct netlink_ext_ack *extack)
1438 struct net *net = sock_net(skb->sk);
1439 struct tcmsg *tcm = nlmsg_data(n);
1440 struct nlattr *tca[TCA_MAX + 1];
1441 struct net_device *dev;
1443 struct Qdisc *q = NULL;
1444 struct Qdisc *p = NULL;
1447 if ((n->nlmsg_type != RTM_GETQDISC) &&
1448 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1451 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1452 rtm_tca_policy, extack);
1456 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1460 clid = tcm->tcm_parent;
1462 if (clid != TC_H_ROOT) {
1463 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
1464 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1466 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified classid");
1469 q = qdisc_leaf(p, clid);
1470 } else if (dev_ingress_queue(dev)) {
1471 q = dev_ingress_queue(dev)->qdisc_sleeping;
1474 q = rtnl_dereference(dev->qdisc);
1477 NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device");
1481 if (tcm->tcm_handle && q->handle != tcm->tcm_handle) {
1482 NL_SET_ERR_MSG(extack, "Invalid handle");
1486 q = qdisc_lookup(dev, tcm->tcm_handle);
1488 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified handle");
1493 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1494 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1498 if (n->nlmsg_type == RTM_DELQDISC) {
1500 NL_SET_ERR_MSG(extack, "Classid cannot be zero");
1503 if (q->handle == 0) {
1504 NL_SET_ERR_MSG(extack, "Cannot delete qdisc with handle of zero");
1507 err = qdisc_graft(dev, p, skb, n, clid, NULL, q, extack);
1511 qdisc_notify(net, skb, n, clid, NULL, q);
1516 static bool req_create_or_replace(struct nlmsghdr *n)
1518 return (n->nlmsg_flags & NLM_F_CREATE &&
1519 n->nlmsg_flags & NLM_F_REPLACE);
1522 static bool req_create_exclusive(struct nlmsghdr *n)
1524 return (n->nlmsg_flags & NLM_F_CREATE &&
1525 n->nlmsg_flags & NLM_F_EXCL);
1528 static bool req_change(struct nlmsghdr *n)
1530 return (!(n->nlmsg_flags & NLM_F_CREATE) &&
1531 !(n->nlmsg_flags & NLM_F_REPLACE) &&
1532 !(n->nlmsg_flags & NLM_F_EXCL));
1536 * Create/change qdisc.
1538 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1539 struct netlink_ext_ack *extack)
1541 struct net *net = sock_net(skb->sk);
1543 struct nlattr *tca[TCA_MAX + 1];
1544 struct net_device *dev;
1546 struct Qdisc *q, *p;
1549 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1553 /* Reinit, just in case something touches this. */
1554 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1555 rtm_tca_policy, extack);
1559 tcm = nlmsg_data(n);
1560 clid = tcm->tcm_parent;
1563 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1569 if (clid != TC_H_ROOT) {
1570 if (clid != TC_H_INGRESS) {
1571 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1573 NL_SET_ERR_MSG(extack, "Failed to find specified qdisc");
1576 q = qdisc_leaf(p, clid);
1577 } else if (dev_ingress_queue_create(dev)) {
1578 q = dev_ingress_queue(dev)->qdisc_sleeping;
1581 q = rtnl_dereference(dev->qdisc);
1584 /* It may be default qdisc, ignore it */
1585 if (q && q->handle == 0)
1588 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1589 if (tcm->tcm_handle) {
1590 if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) {
1591 NL_SET_ERR_MSG(extack, "NLM_F_REPLACE needed to override");
1594 if (TC_H_MIN(tcm->tcm_handle)) {
1595 NL_SET_ERR_MSG(extack, "Invalid minor handle");
1598 q = qdisc_lookup(dev, tcm->tcm_handle);
1600 goto create_n_graft;
1601 if (n->nlmsg_flags & NLM_F_EXCL) {
1602 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot override");
1605 if (tca[TCA_KIND] &&
1606 nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1607 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1610 if (q->flags & TCQ_F_INGRESS) {
1611 NL_SET_ERR_MSG(extack,
1612 "Cannot regraft ingress or clsact Qdiscs");
1616 (p && check_loop(q, p, 0))) {
1617 NL_SET_ERR_MSG(extack, "Qdisc parent/child loop detected");
1620 if (clid == TC_H_INGRESS) {
1621 NL_SET_ERR_MSG(extack, "Ingress cannot graft directly");
1624 qdisc_refcount_inc(q);
1628 goto create_n_graft;
1630 /* This magic test requires explanation.
1632 * We know, that some child q is already
1633 * attached to this parent and have choice:
1634 * 1) change it or 2) create/graft new one.
1635 * If the requested qdisc kind is different
1636 * than the existing one, then we choose graft.
1637 * If they are the same then this is "change"
1638 * operation - just let it fallthrough..
1640 * 1. We are allowed to create/graft only
1641 * if the request is explicitly stating
1642 * "please create if it doesn't exist".
1644 * 2. If the request is to exclusive create
1645 * then the qdisc tcm_handle is not expected
1646 * to exist, so that we choose create/graft too.
1648 * 3. The last case is when no flags are set.
1649 * This will happen when for example tc
1650 * utility issues a "change" command.
1651 * Alas, it is sort of hole in API, we
1652 * cannot decide what to do unambiguously.
1653 * For now we select create/graft.
1655 if (tca[TCA_KIND] &&
1656 nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1657 if (req_create_or_replace(n) ||
1658 req_create_exclusive(n))
1659 goto create_n_graft;
1660 else if (req_change(n))
1661 goto create_n_graft2;
1666 if (!tcm->tcm_handle) {
1667 NL_SET_ERR_MSG(extack, "Handle cannot be zero");
1670 q = qdisc_lookup(dev, tcm->tcm_handle);
1673 /* Change qdisc parameters */
1675 NL_SET_ERR_MSG(extack, "Specified qdisc not found");
1678 if (n->nlmsg_flags & NLM_F_EXCL) {
1679 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot modify");
1682 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1683 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1686 err = qdisc_change(q, tca, extack);
1688 qdisc_notify(net, skb, n, clid, NULL, q);
1692 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1693 NL_SET_ERR_MSG(extack, "Qdisc not found. To create specify NLM_F_CREATE flag");
1697 if (clid == TC_H_INGRESS) {
1698 if (dev_ingress_queue(dev)) {
1699 q = qdisc_create(dev, dev_ingress_queue(dev), p,
1700 tcm->tcm_parent, tcm->tcm_parent,
1703 NL_SET_ERR_MSG(extack, "Cannot find ingress queue for specified device");
1707 struct netdev_queue *dev_queue;
1709 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
1710 dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1712 dev_queue = p->dev_queue;
1714 dev_queue = netdev_get_tx_queue(dev, 0);
1716 q = qdisc_create(dev, dev_queue, p,
1717 tcm->tcm_parent, tcm->tcm_handle,
1727 err = qdisc_graft(dev, p, skb, n, clid, q, NULL, extack);
1737 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1738 struct netlink_callback *cb,
1739 int *q_idx_p, int s_q_idx, bool recur,
1740 bool dump_invisible)
1742 int ret = 0, q_idx = *q_idx_p;
1750 if (q_idx < s_q_idx) {
1753 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1754 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1755 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1761 /* If dumping singletons, there is no qdisc_dev(root) and the singleton
1762 * itself has already been dumped.
1764 * If we've already dumped the top-level (ingress) qdisc above and the global
1765 * qdisc hashtable, we don't want to hit it again
1767 if (!qdisc_dev(root) || !recur)
1770 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
1771 if (q_idx < s_q_idx) {
1775 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1776 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1777 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1791 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1793 struct net *net = sock_net(skb->sk);
1796 struct net_device *dev;
1797 const struct nlmsghdr *nlh = cb->nlh;
1798 struct nlattr *tca[TCA_MAX + 1];
1801 s_idx = cb->args[0];
1802 s_q_idx = q_idx = cb->args[1];
1807 err = nlmsg_parse_deprecated(nlh, sizeof(struct tcmsg), tca, TCA_MAX,
1808 rtm_tca_policy, cb->extack);
1812 for_each_netdev(net, dev) {
1813 struct netdev_queue *dev_queue;
1821 if (tc_dump_qdisc_root(rtnl_dereference(dev->qdisc),
1822 skb, cb, &q_idx, s_q_idx,
1823 true, tca[TCA_DUMP_INVISIBLE]) < 0)
1826 dev_queue = dev_ingress_queue(dev);
1828 tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
1829 &q_idx, s_q_idx, false,
1830 tca[TCA_DUMP_INVISIBLE]) < 0)
1839 cb->args[1] = q_idx;
1846 /************************************************
1847 * Traffic classes manipulation. *
1848 ************************************************/
1850 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1852 u32 portid, u32 seq, u16 flags, int event)
1855 struct nlmsghdr *nlh;
1856 unsigned char *b = skb_tail_pointer(skb);
1858 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1861 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1863 goto out_nlmsg_trim;
1864 tcm = nlmsg_data(nlh);
1865 tcm->tcm_family = AF_UNSPEC;
1868 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1869 tcm->tcm_parent = q->handle;
1870 tcm->tcm_handle = q->handle;
1872 if (nla_put_string(skb, TCA_KIND, q->ops->id))
1873 goto nla_put_failure;
1874 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1875 goto nla_put_failure;
1877 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1878 NULL, &d, TCA_PAD) < 0)
1879 goto nla_put_failure;
1881 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1882 goto nla_put_failure;
1884 if (gnet_stats_finish_copy(&d) < 0)
1885 goto nla_put_failure;
1887 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1896 static int tclass_notify(struct net *net, struct sk_buff *oskb,
1897 struct nlmsghdr *n, struct Qdisc *q,
1898 unsigned long cl, int event)
1900 struct sk_buff *skb;
1901 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1904 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1908 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event) < 0) {
1913 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1914 n->nlmsg_flags & NLM_F_ECHO);
1920 static int tclass_del_notify(struct net *net,
1921 const struct Qdisc_class_ops *cops,
1922 struct sk_buff *oskb, struct nlmsghdr *n,
1923 struct Qdisc *q, unsigned long cl)
1925 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1926 struct sk_buff *skb;
1932 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1936 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,
1937 RTM_DELTCLASS) < 0) {
1942 err = cops->delete(q, cl);
1948 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1949 n->nlmsg_flags & NLM_F_ECHO);
1955 #ifdef CONFIG_NET_CLS
1957 struct tcf_bind_args {
1958 struct tcf_walker w;
1964 static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
1966 struct tcf_bind_args *a = (void *)arg;
1968 if (tp->ops->bind_class) {
1969 struct Qdisc *q = tcf_block_q(tp->chain->block);
1972 tp->ops->bind_class(n, a->classid, a->cl, q, a->base);
1978 struct tc_bind_class_args {
1979 struct qdisc_walker w;
1980 unsigned long new_cl;
1985 static int tc_bind_class_walker(struct Qdisc *q, unsigned long cl,
1986 struct qdisc_walker *w)
1988 struct tc_bind_class_args *a = (struct tc_bind_class_args *)w;
1989 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1990 struct tcf_block *block;
1991 struct tcf_chain *chain;
1993 block = cops->tcf_block(q, cl, NULL);
1996 for (chain = tcf_get_next_chain(block, NULL);
1998 chain = tcf_get_next_chain(block, chain)) {
1999 struct tcf_proto *tp;
2001 for (tp = tcf_get_next_proto(chain, NULL, true);
2002 tp; tp = tcf_get_next_proto(chain, tp, true)) {
2003 struct tcf_bind_args arg = {};
2005 arg.w.fn = tcf_node_bind;
2006 arg.classid = a->clid;
2009 tp->ops->walk(tp, &arg.w, true);
2016 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
2017 unsigned long new_cl)
2019 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
2020 struct tc_bind_class_args args = {};
2022 if (!cops->tcf_block)
2024 args.portid = portid;
2026 args.new_cl = new_cl;
2027 args.w.fn = tc_bind_class_walker;
2028 q->ops->cl_ops->walk(q, &args.w);
2033 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
2034 unsigned long new_cl)
2040 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n,
2041 struct netlink_ext_ack *extack)
2043 struct net *net = sock_net(skb->sk);
2044 struct tcmsg *tcm = nlmsg_data(n);
2045 struct nlattr *tca[TCA_MAX + 1];
2046 struct net_device *dev;
2047 struct Qdisc *q = NULL;
2048 const struct Qdisc_class_ops *cops;
2049 unsigned long cl = 0;
2050 unsigned long new_cl;
2056 if ((n->nlmsg_type != RTM_GETTCLASS) &&
2057 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2060 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
2061 rtm_tca_policy, extack);
2065 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2070 parent == TC_H_UNSPEC - unspecified parent.
2071 parent == TC_H_ROOT - class is root, which has no parent.
2072 parent == X:0 - parent is root class.
2073 parent == X:Y - parent is a node in hierarchy.
2074 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
2076 handle == 0:0 - generate handle from kernel pool.
2077 handle == 0:Y - class is X:Y, where X:0 is qdisc.
2078 handle == X:Y - clear.
2079 handle == X:0 - root class.
2082 /* Step 1. Determine qdisc handle X:0 */
2084 portid = tcm->tcm_parent;
2085 clid = tcm->tcm_handle;
2086 qid = TC_H_MAJ(clid);
2088 if (portid != TC_H_ROOT) {
2089 u32 qid1 = TC_H_MAJ(portid);
2092 /* If both majors are known, they must be identical. */
2097 } else if (qid == 0)
2098 qid = rtnl_dereference(dev->qdisc)->handle;
2100 /* Now qid is genuine qdisc handle consistent
2101 * both with parent and child.
2103 * TC_H_MAJ(portid) still may be unspecified, complete it now.
2106 portid = TC_H_MAKE(qid, portid);
2109 qid = rtnl_dereference(dev->qdisc)->handle;
2112 /* OK. Locate qdisc */
2113 q = qdisc_lookup(dev, qid);
2117 /* An check that it supports classes */
2118 cops = q->ops->cl_ops;
2122 /* Now try to get class */
2124 if (portid == TC_H_ROOT)
2127 clid = TC_H_MAKE(qid, clid);
2130 cl = cops->find(q, clid);
2134 if (n->nlmsg_type != RTM_NEWTCLASS ||
2135 !(n->nlmsg_flags & NLM_F_CREATE))
2138 switch (n->nlmsg_type) {
2141 if (n->nlmsg_flags & NLM_F_EXCL)
2145 err = tclass_del_notify(net, cops, skb, n, q, cl);
2146 /* Unbind the class with flilters with 0 */
2147 tc_bind_tclass(q, portid, clid, 0);
2150 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
2158 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
2159 NL_SET_ERR_MSG(extack, "Shared blocks are not supported for classes");
2166 err = cops->change(q, clid, portid, tca, &new_cl, extack);
2168 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
2169 /* We just create a new class, need to do reverse binding. */
2171 tc_bind_tclass(q, portid, clid, new_cl);
2177 struct qdisc_dump_args {
2178 struct qdisc_walker w;
2179 struct sk_buff *skb;
2180 struct netlink_callback *cb;
2183 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl,
2184 struct qdisc_walker *arg)
2186 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
2188 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid,
2189 a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2193 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
2194 struct tcmsg *tcm, struct netlink_callback *cb,
2197 struct qdisc_dump_args arg;
2199 if (tc_qdisc_dump_ignore(q, false) ||
2200 *t_p < s_t || !q->ops->cl_ops ||
2202 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
2207 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
2208 arg.w.fn = qdisc_class_dump;
2212 arg.w.skip = cb->args[1];
2214 q->ops->cl_ops->walk(q, &arg.w);
2215 cb->args[1] = arg.w.count;
2222 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
2223 struct tcmsg *tcm, struct netlink_callback *cb,
2224 int *t_p, int s_t, bool recur)
2232 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
2235 if (!qdisc_dev(root) || !recur)
2238 if (tcm->tcm_parent) {
2239 q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent));
2240 if (q && q != root &&
2241 tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2245 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
2246 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2253 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
2255 struct tcmsg *tcm = nlmsg_data(cb->nlh);
2256 struct net *net = sock_net(skb->sk);
2257 struct netdev_queue *dev_queue;
2258 struct net_device *dev;
2261 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2263 dev = dev_get_by_index(net, tcm->tcm_ifindex);
2270 if (tc_dump_tclass_root(rtnl_dereference(dev->qdisc),
2271 skb, tcm, cb, &t, s_t, true) < 0)
2274 dev_queue = dev_ingress_queue(dev);
2276 tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb,
2277 &t, s_t, false) < 0)
2287 #ifdef CONFIG_PROC_FS
2288 static int psched_show(struct seq_file *seq, void *v)
2290 seq_printf(seq, "%08x %08x %08x %08x\n",
2291 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
2293 (u32)NSEC_PER_SEC / hrtimer_resolution);
2298 static int __net_init psched_net_init(struct net *net)
2300 struct proc_dir_entry *e;
2302 e = proc_create_single("psched", 0, net->proc_net, psched_show);
2309 static void __net_exit psched_net_exit(struct net *net)
2311 remove_proc_entry("psched", net->proc_net);
2314 static int __net_init psched_net_init(struct net *net)
2319 static void __net_exit psched_net_exit(struct net *net)
2324 static struct pernet_operations psched_net_ops = {
2325 .init = psched_net_init,
2326 .exit = psched_net_exit,
2329 static int __init pktsched_init(void)
2333 err = register_pernet_subsys(&psched_net_ops);
2335 pr_err("pktsched_init: "
2336 "cannot initialize per netns operations\n");
2340 register_qdisc(&pfifo_fast_ops);
2341 register_qdisc(&pfifo_qdisc_ops);
2342 register_qdisc(&bfifo_qdisc_ops);
2343 register_qdisc(&pfifo_head_drop_qdisc_ops);
2344 register_qdisc(&mq_qdisc_ops);
2345 register_qdisc(&noqueue_qdisc_ops);
2347 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, 0);
2348 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, 0);
2349 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc,
2351 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, 0);
2352 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, 0);
2353 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass,
2359 subsys_initcall(pktsched_init);