1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * net/sched/sch_api.c Packet scheduler API.
5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
9 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
10 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
11 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
14 #include <linux/module.h>
15 #include <linux/types.h>
16 #include <linux/kernel.h>
17 #include <linux/string.h>
18 #include <linux/errno.h>
19 #include <linux/skbuff.h>
20 #include <linux/init.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/kmod.h>
24 #include <linux/list.h>
25 #include <linux/hrtimer.h>
26 #include <linux/slab.h>
27 #include <linux/hashtable.h>
29 #include <net/net_namespace.h>
31 #include <net/netlink.h>
32 #include <net/pkt_sched.h>
33 #include <net/pkt_cls.h>
35 #include <trace/events/qdisc.h>
42 This file consists of two interrelated parts:
44 1. queueing disciplines manager frontend.
45 2. traffic classes manager frontend.
47 Generally, queueing discipline ("qdisc") is a black box,
48 which is able to enqueue packets and to dequeue them (when
49 device is ready to send something) in order and at times
50 determined by algorithm hidden in it.
52 qdisc's are divided to two categories:
53 - "queues", which have no internal structure visible from outside.
54 - "schedulers", which split all the packets to "traffic classes",
55 using "packet classifiers" (look at cls_api.c)
57 In turn, classes may have child qdiscs (as rule, queues)
58 attached to them etc. etc. etc.
60 The goal of the routines in this file is to translate
61 information supplied by user in the form of handles
62 to more intelligible for kernel form, to make some sanity
63 checks and part of work, which is common to all qdiscs
64 and to provide rtnetlink notifications.
66 All real intelligent work is done inside qdisc modules.
70 Every discipline has two major routines: enqueue and dequeue.
74 dequeue usually returns a skb to send. It is allowed to return NULL,
75 but it does not mean that queue is empty, it just means that
76 discipline does not want to send anything this time.
77 Queue is really empty if q->q.qlen == 0.
78 For complicated disciplines with multiple queues q->q is not
79 real packet queue, but however q->q.qlen must be valid.
83 enqueue returns 0, if packet was enqueued successfully.
84 If packet (this one or another one) was dropped, it returns
86 NET_XMIT_DROP - this packet dropped
87 Expected action: do not backoff, but wait until queue will clear.
88 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
89 Expected action: backoff or ignore
95 like dequeue but without removing a packet from the queue
99 returns qdisc to initial state: purge all buffers, clear all
100 timers, counters (except for statistics) etc.
104 initializes newly created qdisc.
108 destroys resources allocated by init and during lifetime of qdisc.
112 changes qdisc parameters.
115 /* Protects list of registered TC modules. It is pure SMP lock. */
116 static DEFINE_RWLOCK(qdisc_mod_lock);
119 /************************************************
120 * Queueing disciplines manipulation. *
121 ************************************************/
124 /* The list of all installed queueing disciplines. */
126 static struct Qdisc_ops *qdisc_base;
128 /* Register/unregister queueing discipline */
130 int register_qdisc(struct Qdisc_ops *qops)
132 struct Qdisc_ops *q, **qp;
135 write_lock(&qdisc_mod_lock);
136 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
137 if (!strcmp(qops->id, q->id))
140 if (qops->enqueue == NULL)
141 qops->enqueue = noop_qdisc_ops.enqueue;
142 if (qops->peek == NULL) {
143 if (qops->dequeue == NULL)
144 qops->peek = noop_qdisc_ops.peek;
148 if (qops->dequeue == NULL)
149 qops->dequeue = noop_qdisc_ops.dequeue;
152 const struct Qdisc_class_ops *cops = qops->cl_ops;
154 if (!(cops->find && cops->walk && cops->leaf))
157 if (cops->tcf_block && !(cops->bind_tcf && cops->unbind_tcf))
165 write_unlock(&qdisc_mod_lock);
172 EXPORT_SYMBOL(register_qdisc);
174 void unregister_qdisc(struct Qdisc_ops *qops)
176 struct Qdisc_ops *q, **qp;
179 write_lock(&qdisc_mod_lock);
180 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
188 write_unlock(&qdisc_mod_lock);
190 WARN(err, "unregister qdisc(%s) failed\n", qops->id);
192 EXPORT_SYMBOL(unregister_qdisc);
194 /* Get default qdisc if not otherwise specified */
195 void qdisc_get_default(char *name, size_t len)
197 read_lock(&qdisc_mod_lock);
198 strscpy(name, default_qdisc_ops->id, len);
199 read_unlock(&qdisc_mod_lock);
202 static struct Qdisc_ops *qdisc_lookup_default(const char *name)
204 struct Qdisc_ops *q = NULL;
206 for (q = qdisc_base; q; q = q->next) {
207 if (!strcmp(name, q->id)) {
208 if (!try_module_get(q->owner))
217 /* Set new default qdisc to use */
218 int qdisc_set_default(const char *name)
220 const struct Qdisc_ops *ops;
222 if (!capable(CAP_NET_ADMIN))
225 write_lock(&qdisc_mod_lock);
226 ops = qdisc_lookup_default(name);
228 /* Not found, drop lock and try to load module */
229 write_unlock(&qdisc_mod_lock);
230 request_module("sch_%s", name);
231 write_lock(&qdisc_mod_lock);
233 ops = qdisc_lookup_default(name);
237 /* Set new default */
238 module_put(default_qdisc_ops->owner);
239 default_qdisc_ops = ops;
241 write_unlock(&qdisc_mod_lock);
243 return ops ? 0 : -ENOENT;
246 #ifdef CONFIG_NET_SCH_DEFAULT
247 /* Set default value from kernel config */
248 static int __init sch_default_qdisc(void)
250 return qdisc_set_default(CONFIG_DEFAULT_NET_SCH);
252 late_initcall(sch_default_qdisc);
255 /* We know handle. Find qdisc among all qdisc's attached to device
256 * (root qdisc, all its children, children of children etc.)
257 * Note: caller either uses rtnl or rcu_read_lock()
260 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
264 if (!qdisc_dev(root))
265 return (root->handle == handle ? root : NULL);
267 if (!(root->flags & TCQ_F_BUILTIN) &&
268 root->handle == handle)
271 hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle,
272 lockdep_rtnl_is_held()) {
273 if (q->handle == handle)
279 void qdisc_hash_add(struct Qdisc *q, bool invisible)
281 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
283 hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);
285 q->flags |= TCQ_F_INVISIBLE;
288 EXPORT_SYMBOL(qdisc_hash_add);
290 void qdisc_hash_del(struct Qdisc *q)
292 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
294 hash_del_rcu(&q->hash);
297 EXPORT_SYMBOL(qdisc_hash_del);
299 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
305 q = qdisc_match_from_root(rtnl_dereference(dev->qdisc), handle);
309 if (dev_ingress_queue(dev))
310 q = qdisc_match_from_root(
311 dev_ingress_queue(dev)->qdisc_sleeping,
317 struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle)
319 struct netdev_queue *nq;
324 q = qdisc_match_from_root(rcu_dereference(dev->qdisc), handle);
328 nq = dev_ingress_queue_rcu(dev);
330 q = qdisc_match_from_root(nq->qdisc_sleeping, handle);
335 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
338 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
342 cl = cops->find(p, classid);
346 return cops->leaf(p, cl);
349 /* Find queueing discipline by name */
351 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
353 struct Qdisc_ops *q = NULL;
356 read_lock(&qdisc_mod_lock);
357 for (q = qdisc_base; q; q = q->next) {
358 if (nla_strcmp(kind, q->id) == 0) {
359 if (!try_module_get(q->owner))
364 read_unlock(&qdisc_mod_lock);
369 /* The linklayer setting were not transferred from iproute2, in older
370 * versions, and the rate tables lookup systems have been dropped in
371 * the kernel. To keep backward compatible with older iproute2 tc
372 * utils, we detect the linklayer setting by detecting if the rate
373 * table were modified.
375 * For linklayer ATM table entries, the rate table will be aligned to
376 * 48 bytes, thus some table entries will contain the same value. The
377 * mpu (min packet unit) is also encoded into the old rate table, thus
378 * starting from the mpu, we find low and high table entries for
379 * mapping this cell. If these entries contain the same value, when
380 * the rate tables have been modified for linklayer ATM.
382 * This is done by rounding mpu to the nearest 48 bytes cell/entry,
383 * and then roundup to the next cell, calc the table entry one below,
386 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab)
388 int low = roundup(r->mpu, 48);
389 int high = roundup(low+1, 48);
390 int cell_low = low >> r->cell_log;
391 int cell_high = (high >> r->cell_log) - 1;
393 /* rtab is too inaccurate at rates > 100Mbit/s */
394 if ((r->rate > (100000000/8)) || (rtab[0] == 0)) {
395 pr_debug("TC linklayer: Giving up ATM detection\n");
396 return TC_LINKLAYER_ETHERNET;
399 if ((cell_high > cell_low) && (cell_high < 256)
400 && (rtab[cell_low] == rtab[cell_high])) {
401 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n",
402 cell_low, cell_high, rtab[cell_high]);
403 return TC_LINKLAYER_ATM;
405 return TC_LINKLAYER_ETHERNET;
408 static struct qdisc_rate_table *qdisc_rtab_list;
410 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,
412 struct netlink_ext_ack *extack)
414 struct qdisc_rate_table *rtab;
416 if (tab == NULL || r->rate == 0 ||
417 r->cell_log == 0 || r->cell_log >= 32 ||
418 nla_len(tab) != TC_RTAB_SIZE) {
419 NL_SET_ERR_MSG(extack, "Invalid rate table parameters for searching");
423 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
424 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) &&
425 !memcmp(&rtab->data, nla_data(tab), 1024)) {
431 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
435 memcpy(rtab->data, nla_data(tab), 1024);
436 if (r->linklayer == TC_LINKLAYER_UNAWARE)
437 r->linklayer = __detect_linklayer(r, rtab->data);
438 rtab->next = qdisc_rtab_list;
439 qdisc_rtab_list = rtab;
441 NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table");
445 EXPORT_SYMBOL(qdisc_get_rtab);
447 void qdisc_put_rtab(struct qdisc_rate_table *tab)
449 struct qdisc_rate_table *rtab, **rtabp;
451 if (!tab || --tab->refcnt)
454 for (rtabp = &qdisc_rtab_list;
455 (rtab = *rtabp) != NULL;
456 rtabp = &rtab->next) {
464 EXPORT_SYMBOL(qdisc_put_rtab);
466 static LIST_HEAD(qdisc_stab_list);
468 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
469 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
470 [TCA_STAB_DATA] = { .type = NLA_BINARY },
473 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt,
474 struct netlink_ext_ack *extack)
476 struct nlattr *tb[TCA_STAB_MAX + 1];
477 struct qdisc_size_table *stab;
478 struct tc_sizespec *s;
479 unsigned int tsize = 0;
483 err = nla_parse_nested_deprecated(tb, TCA_STAB_MAX, opt, stab_policy,
487 if (!tb[TCA_STAB_BASE]) {
488 NL_SET_ERR_MSG(extack, "Size table base attribute is missing");
489 return ERR_PTR(-EINVAL);
492 s = nla_data(tb[TCA_STAB_BASE]);
495 if (!tb[TCA_STAB_DATA]) {
496 NL_SET_ERR_MSG(extack, "Size table data attribute is missing");
497 return ERR_PTR(-EINVAL);
499 tab = nla_data(tb[TCA_STAB_DATA]);
500 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
503 if (tsize != s->tsize || (!tab && tsize > 0)) {
504 NL_SET_ERR_MSG(extack, "Invalid size of size table");
505 return ERR_PTR(-EINVAL);
508 list_for_each_entry(stab, &qdisc_stab_list, list) {
509 if (memcmp(&stab->szopts, s, sizeof(*s)))
512 memcmp(stab->data, tab, flex_array_size(stab, data, tsize)))
518 if (s->size_log > STAB_SIZE_LOG_MAX ||
519 s->cell_log > STAB_SIZE_LOG_MAX) {
520 NL_SET_ERR_MSG(extack, "Invalid logarithmic size of size table");
521 return ERR_PTR(-EINVAL);
524 stab = kmalloc(struct_size(stab, data, tsize), GFP_KERNEL);
526 return ERR_PTR(-ENOMEM);
531 memcpy(stab->data, tab, flex_array_size(stab, data, tsize));
533 list_add_tail(&stab->list, &qdisc_stab_list);
538 void qdisc_put_stab(struct qdisc_size_table *tab)
543 if (--tab->refcnt == 0) {
544 list_del(&tab->list);
548 EXPORT_SYMBOL(qdisc_put_stab);
550 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
554 nest = nla_nest_start_noflag(skb, TCA_STAB);
556 goto nla_put_failure;
557 if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts))
558 goto nla_put_failure;
559 nla_nest_end(skb, nest);
567 void __qdisc_calculate_pkt_len(struct sk_buff *skb,
568 const struct qdisc_size_table *stab)
572 pkt_len = skb->len + stab->szopts.overhead;
573 if (unlikely(!stab->szopts.tsize))
576 slot = pkt_len + stab->szopts.cell_align;
577 if (unlikely(slot < 0))
580 slot >>= stab->szopts.cell_log;
581 if (likely(slot < stab->szopts.tsize))
582 pkt_len = stab->data[slot];
584 pkt_len = stab->data[stab->szopts.tsize - 1] *
585 (slot / stab->szopts.tsize) +
586 stab->data[slot % stab->szopts.tsize];
588 pkt_len <<= stab->szopts.size_log;
590 if (unlikely(pkt_len < 1))
592 qdisc_skb_cb(skb)->pkt_len = pkt_len;
594 EXPORT_SYMBOL(__qdisc_calculate_pkt_len);
596 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc)
598 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
599 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n",
600 txt, qdisc->ops->id, qdisc->handle >> 16);
601 qdisc->flags |= TCQ_F_WARN_NONWC;
604 EXPORT_SYMBOL(qdisc_warn_nonwc);
606 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
608 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
612 __netif_schedule(qdisc_root(wd->qdisc));
615 return HRTIMER_NORESTART;
618 void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc,
621 hrtimer_init(&wd->timer, clockid, HRTIMER_MODE_ABS_PINNED);
622 wd->timer.function = qdisc_watchdog;
625 EXPORT_SYMBOL(qdisc_watchdog_init_clockid);
627 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
629 qdisc_watchdog_init_clockid(wd, qdisc, CLOCK_MONOTONIC);
631 EXPORT_SYMBOL(qdisc_watchdog_init);
633 void qdisc_watchdog_schedule_range_ns(struct qdisc_watchdog *wd, u64 expires,
636 if (test_bit(__QDISC_STATE_DEACTIVATED,
637 &qdisc_root_sleeping(wd->qdisc)->state))
640 if (hrtimer_is_queued(&wd->timer)) {
641 /* If timer is already set in [expires, expires + delta_ns],
642 * do not reprogram it.
644 if (wd->last_expires - expires <= delta_ns)
648 wd->last_expires = expires;
649 hrtimer_start_range_ns(&wd->timer,
650 ns_to_ktime(expires),
652 HRTIMER_MODE_ABS_PINNED);
654 EXPORT_SYMBOL(qdisc_watchdog_schedule_range_ns);
656 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
658 hrtimer_cancel(&wd->timer);
660 EXPORT_SYMBOL(qdisc_watchdog_cancel);
662 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
664 struct hlist_head *h;
667 h = kvmalloc_array(n, sizeof(struct hlist_head), GFP_KERNEL);
670 for (i = 0; i < n; i++)
671 INIT_HLIST_HEAD(&h[i]);
676 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
678 struct Qdisc_class_common *cl;
679 struct hlist_node *next;
680 struct hlist_head *nhash, *ohash;
681 unsigned int nsize, nmask, osize;
684 /* Rehash when load factor exceeds 0.75 */
685 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
687 nsize = clhash->hashsize * 2;
689 nhash = qdisc_class_hash_alloc(nsize);
693 ohash = clhash->hash;
694 osize = clhash->hashsize;
697 for (i = 0; i < osize; i++) {
698 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) {
699 h = qdisc_class_hash(cl->classid, nmask);
700 hlist_add_head(&cl->hnode, &nhash[h]);
703 clhash->hash = nhash;
704 clhash->hashsize = nsize;
705 clhash->hashmask = nmask;
706 sch_tree_unlock(sch);
710 EXPORT_SYMBOL(qdisc_class_hash_grow);
712 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
714 unsigned int size = 4;
716 clhash->hash = qdisc_class_hash_alloc(size);
719 clhash->hashsize = size;
720 clhash->hashmask = size - 1;
721 clhash->hashelems = 0;
724 EXPORT_SYMBOL(qdisc_class_hash_init);
726 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
728 kvfree(clhash->hash);
730 EXPORT_SYMBOL(qdisc_class_hash_destroy);
732 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
733 struct Qdisc_class_common *cl)
737 INIT_HLIST_NODE(&cl->hnode);
738 h = qdisc_class_hash(cl->classid, clhash->hashmask);
739 hlist_add_head(&cl->hnode, &clhash->hash[h]);
742 EXPORT_SYMBOL(qdisc_class_hash_insert);
744 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
745 struct Qdisc_class_common *cl)
747 hlist_del(&cl->hnode);
750 EXPORT_SYMBOL(qdisc_class_hash_remove);
752 /* Allocate an unique handle from space managed by kernel
753 * Possible range is [8000-FFFF]:0000 (0x8000 values)
755 static u32 qdisc_alloc_handle(struct net_device *dev)
758 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
761 autohandle += TC_H_MAKE(0x10000U, 0);
762 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
763 autohandle = TC_H_MAKE(0x80000000U, 0);
764 if (!qdisc_lookup(dev, autohandle))
772 void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len)
774 bool qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED;
775 const struct Qdisc_class_ops *cops;
781 if (n == 0 && len == 0)
783 drops = max_t(int, n, 0);
785 while ((parentid = sch->parent)) {
786 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
789 if (sch->flags & TCQ_F_NOPARENT)
791 /* Notify parent qdisc only if child qdisc becomes empty.
793 * If child was empty even before update then backlog
794 * counter is screwed and we skip notification because
795 * parent class is already passive.
797 * If the original child was offloaded then it is allowed
798 * to be seem as empty, so the parent is notified anyway.
800 notify = !sch->q.qlen && !WARN_ON_ONCE(!n &&
801 !qdisc_is_offloaded);
802 /* TODO: perform the search on a per txq basis */
803 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
805 WARN_ON_ONCE(parentid != TC_H_ROOT);
808 cops = sch->ops->cl_ops;
809 if (notify && cops->qlen_notify) {
810 cl = cops->find(sch, parentid);
811 cops->qlen_notify(sch, cl);
814 sch->qstats.backlog -= len;
815 __qdisc_qstats_drop(sch, drops);
819 EXPORT_SYMBOL(qdisc_tree_reduce_backlog);
821 int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type,
824 struct net_device *dev = qdisc_dev(sch);
827 sch->flags &= ~TCQ_F_OFFLOADED;
828 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
831 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
832 if (err == -EOPNOTSUPP)
836 sch->flags |= TCQ_F_OFFLOADED;
840 EXPORT_SYMBOL(qdisc_offload_dump_helper);
842 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
843 struct Qdisc *new, struct Qdisc *old,
844 enum tc_setup_type type, void *type_data,
845 struct netlink_ext_ack *extack)
847 bool any_qdisc_is_offloaded;
850 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
853 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
855 /* Don't report error if the graft is part of destroy operation. */
856 if (!err || !new || new == &noop_qdisc)
859 /* Don't report error if the parent, the old child and the new
860 * one are not offloaded.
862 any_qdisc_is_offloaded = new->flags & TCQ_F_OFFLOADED;
863 any_qdisc_is_offloaded |= sch && sch->flags & TCQ_F_OFFLOADED;
864 any_qdisc_is_offloaded |= old && old->flags & TCQ_F_OFFLOADED;
866 if (any_qdisc_is_offloaded)
867 NL_SET_ERR_MSG(extack, "Offloading graft operation failed.");
869 EXPORT_SYMBOL(qdisc_offload_graft_helper);
871 void qdisc_offload_query_caps(struct net_device *dev,
872 enum tc_setup_type type,
873 void *caps, size_t caps_len)
875 const struct net_device_ops *ops = dev->netdev_ops;
876 struct tc_query_caps_base base = {
881 memset(caps, 0, caps_len);
883 if (ops->ndo_setup_tc)
884 ops->ndo_setup_tc(dev, TC_QUERY_CAPS, &base);
886 EXPORT_SYMBOL(qdisc_offload_query_caps);
888 static void qdisc_offload_graft_root(struct net_device *dev,
889 struct Qdisc *new, struct Qdisc *old,
890 struct netlink_ext_ack *extack)
892 struct tc_root_qopt_offload graft_offload = {
893 .command = TC_ROOT_GRAFT,
894 .handle = new ? new->handle : 0,
895 .ingress = (new && new->flags & TCQ_F_INGRESS) ||
896 (old && old->flags & TCQ_F_INGRESS),
899 qdisc_offload_graft_helper(dev, NULL, new, old,
900 TC_SETUP_ROOT_QDISC, &graft_offload, extack);
903 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
904 u32 portid, u32 seq, u16 flags, int event)
906 struct gnet_stats_basic_sync __percpu *cpu_bstats = NULL;
907 struct gnet_stats_queue __percpu *cpu_qstats = NULL;
909 struct nlmsghdr *nlh;
910 unsigned char *b = skb_tail_pointer(skb);
912 struct qdisc_size_table *stab;
917 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
920 tcm = nlmsg_data(nlh);
921 tcm->tcm_family = AF_UNSPEC;
924 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
925 tcm->tcm_parent = clid;
926 tcm->tcm_handle = q->handle;
927 tcm->tcm_info = refcount_read(&q->refcnt);
928 if (nla_put_string(skb, TCA_KIND, q->ops->id))
929 goto nla_put_failure;
930 if (q->ops->ingress_block_get) {
931 block_index = q->ops->ingress_block_get(q);
933 nla_put_u32(skb, TCA_INGRESS_BLOCK, block_index))
934 goto nla_put_failure;
936 if (q->ops->egress_block_get) {
937 block_index = q->ops->egress_block_get(q);
939 nla_put_u32(skb, TCA_EGRESS_BLOCK, block_index))
940 goto nla_put_failure;
942 if (q->ops->dump && q->ops->dump(q, skb) < 0)
943 goto nla_put_failure;
944 if (nla_put_u8(skb, TCA_HW_OFFLOAD, !!(q->flags & TCQ_F_OFFLOADED)))
945 goto nla_put_failure;
946 qlen = qdisc_qlen_sum(q);
948 stab = rtnl_dereference(q->stab);
949 if (stab && qdisc_dump_stab(skb, stab) < 0)
950 goto nla_put_failure;
952 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
953 NULL, &d, TCA_PAD) < 0)
954 goto nla_put_failure;
956 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
957 goto nla_put_failure;
959 if (qdisc_is_percpu_stats(q)) {
960 cpu_bstats = q->cpu_bstats;
961 cpu_qstats = q->cpu_qstats;
964 if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats, true) < 0 ||
965 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||
966 gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
967 goto nla_put_failure;
969 if (gnet_stats_finish_copy(&d) < 0)
970 goto nla_put_failure;
972 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
981 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible)
983 if (q->flags & TCQ_F_BUILTIN)
985 if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible)
991 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
992 struct nlmsghdr *n, u32 clid,
993 struct Qdisc *old, struct Qdisc *new)
996 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
998 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1002 if (old && !tc_qdisc_dump_ignore(old, false)) {
1003 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
1004 0, RTM_DELQDISC) < 0)
1007 if (new && !tc_qdisc_dump_ignore(new, false)) {
1008 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
1009 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
1014 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1015 n->nlmsg_flags & NLM_F_ECHO);
1022 static void notify_and_destroy(struct net *net, struct sk_buff *skb,
1023 struct nlmsghdr *n, u32 clid,
1024 struct Qdisc *old, struct Qdisc *new)
1027 qdisc_notify(net, skb, n, clid, old, new);
1033 static void qdisc_clear_nolock(struct Qdisc *sch)
1035 sch->flags &= ~TCQ_F_NOLOCK;
1036 if (!(sch->flags & TCQ_F_CPUSTATS))
1039 free_percpu(sch->cpu_bstats);
1040 free_percpu(sch->cpu_qstats);
1041 sch->cpu_bstats = NULL;
1042 sch->cpu_qstats = NULL;
1043 sch->flags &= ~TCQ_F_CPUSTATS;
1046 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
1049 * When appropriate send a netlink notification using 'skb'
1052 * On success, destroy old qdisc.
1055 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
1056 struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
1057 struct Qdisc *new, struct Qdisc *old,
1058 struct netlink_ext_ack *extack)
1060 struct Qdisc *q = old;
1061 struct net *net = dev_net(dev);
1063 if (parent == NULL) {
1064 unsigned int i, num_q, ingress;
1067 num_q = dev->num_tx_queues;
1068 if ((q && q->flags & TCQ_F_INGRESS) ||
1069 (new && new->flags & TCQ_F_INGRESS)) {
1072 if (!dev_ingress_queue(dev)) {
1073 NL_SET_ERR_MSG(extack, "Device does not have an ingress queue");
1078 if (dev->flags & IFF_UP)
1079 dev_deactivate(dev);
1081 qdisc_offload_graft_root(dev, new, old, extack);
1083 if (new && new->ops->attach && !ingress)
1086 for (i = 0; i < num_q; i++) {
1087 struct netdev_queue *dev_queue = dev_ingress_queue(dev);
1090 dev_queue = netdev_get_tx_queue(dev, i);
1092 old = dev_graft_qdisc(dev_queue, new);
1094 qdisc_refcount_inc(new);
1102 old = rtnl_dereference(dev->qdisc);
1103 if (new && !new->ops->attach)
1104 qdisc_refcount_inc(new);
1105 rcu_assign_pointer(dev->qdisc, new ? : &noop_qdisc);
1107 notify_and_destroy(net, skb, n, classid, old, new);
1109 if (new && new->ops->attach)
1110 new->ops->attach(new);
1112 notify_and_destroy(net, skb, n, classid, old, new);
1115 if (dev->flags & IFF_UP)
1118 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
1122 /* Only support running class lockless if parent is lockless */
1123 if (new && (new->flags & TCQ_F_NOLOCK) && !(parent->flags & TCQ_F_NOLOCK))
1124 qdisc_clear_nolock(new);
1126 if (!cops || !cops->graft)
1129 cl = cops->find(parent, classid);
1131 NL_SET_ERR_MSG(extack, "Specified class not found");
1135 if (new && new->ops == &noqueue_qdisc_ops) {
1136 NL_SET_ERR_MSG(extack, "Cannot assign noqueue to a class");
1140 err = cops->graft(parent, cl, new, &old, extack);
1143 notify_and_destroy(net, skb, n, classid, old, new);
1148 static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca,
1149 struct netlink_ext_ack *extack)
1153 if (tca[TCA_INGRESS_BLOCK]) {
1154 block_index = nla_get_u32(tca[TCA_INGRESS_BLOCK]);
1157 NL_SET_ERR_MSG(extack, "Ingress block index cannot be 0");
1160 if (!sch->ops->ingress_block_set) {
1161 NL_SET_ERR_MSG(extack, "Ingress block sharing is not supported");
1164 sch->ops->ingress_block_set(sch, block_index);
1166 if (tca[TCA_EGRESS_BLOCK]) {
1167 block_index = nla_get_u32(tca[TCA_EGRESS_BLOCK]);
1170 NL_SET_ERR_MSG(extack, "Egress block index cannot be 0");
1173 if (!sch->ops->egress_block_set) {
1174 NL_SET_ERR_MSG(extack, "Egress block sharing is not supported");
1177 sch->ops->egress_block_set(sch, block_index);
1183 Allocate and initialize new qdisc.
1185 Parameters are passed via opt.
1188 static struct Qdisc *qdisc_create(struct net_device *dev,
1189 struct netdev_queue *dev_queue,
1190 u32 parent, u32 handle,
1191 struct nlattr **tca, int *errp,
1192 struct netlink_ext_ack *extack)
1195 struct nlattr *kind = tca[TCA_KIND];
1197 struct Qdisc_ops *ops;
1198 struct qdisc_size_table *stab;
1200 ops = qdisc_lookup_ops(kind);
1201 #ifdef CONFIG_MODULES
1202 if (ops == NULL && kind != NULL) {
1203 char name[IFNAMSIZ];
1204 if (nla_strscpy(name, kind, IFNAMSIZ) >= 0) {
1205 /* We dropped the RTNL semaphore in order to
1206 * perform the module load. So, even if we
1207 * succeeded in loading the module we have to
1208 * tell the caller to replay the request. We
1209 * indicate this using -EAGAIN.
1210 * We replay the request because the device may
1211 * go away in the mean time.
1214 request_module("sch_%s", name);
1216 ops = qdisc_lookup_ops(kind);
1218 /* We will try again qdisc_lookup_ops,
1219 * so don't keep a reference.
1221 module_put(ops->owner);
1231 NL_SET_ERR_MSG(extack, "Specified qdisc kind is unknown");
1235 sch = qdisc_alloc(dev_queue, ops, extack);
1241 sch->parent = parent;
1243 if (handle == TC_H_INGRESS) {
1244 sch->flags |= TCQ_F_INGRESS;
1245 handle = TC_H_MAKE(TC_H_INGRESS, 0);
1248 handle = qdisc_alloc_handle(dev);
1250 NL_SET_ERR_MSG(extack, "Maximum number of qdisc handles was exceeded");
1255 if (!netif_is_multiqueue(dev))
1256 sch->flags |= TCQ_F_ONETXQUEUE;
1259 sch->handle = handle;
1261 /* This exist to keep backward compatible with a userspace
1262 * loophole, what allowed userspace to get IFF_NO_QUEUE
1263 * facility on older kernels by setting tx_queue_len=0 (prior
1264 * to qdisc init), and then forgot to reinit tx_queue_len
1265 * before again attaching a qdisc.
1267 if ((dev->priv_flags & IFF_NO_QUEUE) && (dev->tx_queue_len == 0)) {
1268 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
1269 netdev_info(dev, "Caught tx_queue_len zero misconfig\n");
1272 err = qdisc_block_indexes_set(sch, tca, extack);
1277 err = ops->init(sch, tca[TCA_OPTIONS], extack);
1282 if (tca[TCA_STAB]) {
1283 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1285 err = PTR_ERR(stab);
1288 rcu_assign_pointer(sch->stab, stab);
1290 if (tca[TCA_RATE]) {
1292 if (sch->flags & TCQ_F_MQROOT) {
1293 NL_SET_ERR_MSG(extack, "Cannot attach rate estimator to a multi-queue root qdisc");
1297 err = gen_new_estimator(&sch->bstats,
1304 NL_SET_ERR_MSG(extack, "Failed to generate new estimator");
1309 qdisc_hash_add(sch, false);
1310 trace_qdisc_create(ops, dev, parent);
1315 /* ops->init() failed, we call ->destroy() like qdisc_create_dflt() */
1319 netdev_put(dev, &sch->dev_tracker);
1322 module_put(ops->owner);
1329 * Any broken qdiscs that would require a ops->reset() here?
1330 * The qdisc was never in action so it shouldn't be necessary.
1332 qdisc_put_stab(rtnl_dereference(sch->stab));
1338 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca,
1339 struct netlink_ext_ack *extack)
1341 struct qdisc_size_table *ostab, *stab = NULL;
1344 if (tca[TCA_OPTIONS]) {
1345 if (!sch->ops->change) {
1346 NL_SET_ERR_MSG(extack, "Change operation not supported by specified qdisc");
1349 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
1350 NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
1353 err = sch->ops->change(sch, tca[TCA_OPTIONS], extack);
1358 if (tca[TCA_STAB]) {
1359 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1361 return PTR_ERR(stab);
1364 ostab = rtnl_dereference(sch->stab);
1365 rcu_assign_pointer(sch->stab, stab);
1366 qdisc_put_stab(ostab);
1368 if (tca[TCA_RATE]) {
1369 /* NB: ignores errors from replace_estimator
1370 because change can't be undone. */
1371 if (sch->flags & TCQ_F_MQROOT)
1373 gen_replace_estimator(&sch->bstats,
1384 struct check_loop_arg {
1385 struct qdisc_walker w;
1390 static int check_loop_fn(struct Qdisc *q, unsigned long cl,
1391 struct qdisc_walker *w);
1393 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
1395 struct check_loop_arg arg;
1397 if (q->ops->cl_ops == NULL)
1400 arg.w.stop = arg.w.skip = arg.w.count = 0;
1401 arg.w.fn = check_loop_fn;
1404 q->ops->cl_ops->walk(q, &arg.w);
1405 return arg.w.stop ? -ELOOP : 0;
1409 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
1412 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1413 struct check_loop_arg *arg = (struct check_loop_arg *)w;
1415 leaf = cops->leaf(q, cl);
1417 if (leaf == arg->p || arg->depth > 7)
1419 return check_loop(leaf, arg->p, arg->depth + 1);
1424 const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = {
1425 [TCA_KIND] = { .type = NLA_STRING },
1426 [TCA_RATE] = { .type = NLA_BINARY,
1427 .len = sizeof(struct tc_estimator) },
1428 [TCA_STAB] = { .type = NLA_NESTED },
1429 [TCA_DUMP_INVISIBLE] = { .type = NLA_FLAG },
1430 [TCA_CHAIN] = { .type = NLA_U32 },
1431 [TCA_INGRESS_BLOCK] = { .type = NLA_U32 },
1432 [TCA_EGRESS_BLOCK] = { .type = NLA_U32 },
1439 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1440 struct netlink_ext_ack *extack)
1442 struct net *net = sock_net(skb->sk);
1443 struct tcmsg *tcm = nlmsg_data(n);
1444 struct nlattr *tca[TCA_MAX + 1];
1445 struct net_device *dev;
1447 struct Qdisc *q = NULL;
1448 struct Qdisc *p = NULL;
1451 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1452 rtm_tca_policy, extack);
1456 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1460 clid = tcm->tcm_parent;
1462 if (clid != TC_H_ROOT) {
1463 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
1464 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1466 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified classid");
1469 q = qdisc_leaf(p, clid);
1470 } else if (dev_ingress_queue(dev)) {
1471 q = dev_ingress_queue(dev)->qdisc_sleeping;
1474 q = rtnl_dereference(dev->qdisc);
1477 NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device");
1481 if (tcm->tcm_handle && q->handle != tcm->tcm_handle) {
1482 NL_SET_ERR_MSG(extack, "Invalid handle");
1486 q = qdisc_lookup(dev, tcm->tcm_handle);
1488 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified handle");
1493 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1494 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1498 if (n->nlmsg_type == RTM_DELQDISC) {
1500 NL_SET_ERR_MSG(extack, "Classid cannot be zero");
1503 if (q->handle == 0) {
1504 NL_SET_ERR_MSG(extack, "Cannot delete qdisc with handle of zero");
1507 err = qdisc_graft(dev, p, skb, n, clid, NULL, q, extack);
1511 qdisc_notify(net, skb, n, clid, NULL, q);
1517 * Create/change qdisc.
1520 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1521 struct netlink_ext_ack *extack)
1523 struct net *net = sock_net(skb->sk);
1525 struct nlattr *tca[TCA_MAX + 1];
1526 struct net_device *dev;
1528 struct Qdisc *q, *p;
1532 /* Reinit, just in case something touches this. */
1533 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1534 rtm_tca_policy, extack);
1538 tcm = nlmsg_data(n);
1539 clid = tcm->tcm_parent;
1542 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1548 if (clid != TC_H_ROOT) {
1549 if (clid != TC_H_INGRESS) {
1550 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1552 NL_SET_ERR_MSG(extack, "Failed to find specified qdisc");
1555 q = qdisc_leaf(p, clid);
1556 } else if (dev_ingress_queue_create(dev)) {
1557 q = dev_ingress_queue(dev)->qdisc_sleeping;
1560 q = rtnl_dereference(dev->qdisc);
1563 /* It may be default qdisc, ignore it */
1564 if (q && q->handle == 0)
1567 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1568 if (tcm->tcm_handle) {
1569 if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) {
1570 NL_SET_ERR_MSG(extack, "NLM_F_REPLACE needed to override");
1573 if (TC_H_MIN(tcm->tcm_handle)) {
1574 NL_SET_ERR_MSG(extack, "Invalid minor handle");
1577 q = qdisc_lookup(dev, tcm->tcm_handle);
1579 goto create_n_graft;
1580 if (n->nlmsg_flags & NLM_F_EXCL) {
1581 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot override");
1584 if (tca[TCA_KIND] &&
1585 nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1586 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1590 (p && check_loop(q, p, 0))) {
1591 NL_SET_ERR_MSG(extack, "Qdisc parent/child loop detected");
1594 qdisc_refcount_inc(q);
1598 goto create_n_graft;
1600 /* This magic test requires explanation.
1602 * We know, that some child q is already
1603 * attached to this parent and have choice:
1604 * either to change it or to create/graft new one.
1606 * 1. We are allowed to create/graft only
1607 * if CREATE and REPLACE flags are set.
1609 * 2. If EXCL is set, requestor wanted to say,
1610 * that qdisc tcm_handle is not expected
1611 * to exist, so that we choose create/graft too.
1613 * 3. The last case is when no flags are set.
1614 * Alas, it is sort of hole in API, we
1615 * cannot decide what to do unambiguously.
1616 * For now we select create/graft, if
1617 * user gave KIND, which does not match existing.
1619 if ((n->nlmsg_flags & NLM_F_CREATE) &&
1620 (n->nlmsg_flags & NLM_F_REPLACE) &&
1621 ((n->nlmsg_flags & NLM_F_EXCL) ||
1623 nla_strcmp(tca[TCA_KIND], q->ops->id))))
1624 goto create_n_graft;
1628 if (!tcm->tcm_handle) {
1629 NL_SET_ERR_MSG(extack, "Handle cannot be zero");
1632 q = qdisc_lookup(dev, tcm->tcm_handle);
1635 /* Change qdisc parameters */
1637 NL_SET_ERR_MSG(extack, "Specified qdisc not found");
1640 if (n->nlmsg_flags & NLM_F_EXCL) {
1641 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot modify");
1644 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1645 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1648 err = qdisc_change(q, tca, extack);
1650 qdisc_notify(net, skb, n, clid, NULL, q);
1654 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1655 NL_SET_ERR_MSG(extack, "Qdisc not found. To create specify NLM_F_CREATE flag");
1658 if (clid == TC_H_INGRESS) {
1659 if (dev_ingress_queue(dev)) {
1660 q = qdisc_create(dev, dev_ingress_queue(dev),
1661 tcm->tcm_parent, tcm->tcm_parent,
1664 NL_SET_ERR_MSG(extack, "Cannot find ingress queue for specified device");
1668 struct netdev_queue *dev_queue;
1670 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
1671 dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1673 dev_queue = p->dev_queue;
1675 dev_queue = netdev_get_tx_queue(dev, 0);
1677 q = qdisc_create(dev, dev_queue,
1678 tcm->tcm_parent, tcm->tcm_handle,
1688 err = qdisc_graft(dev, p, skb, n, clid, q, NULL, extack);
1698 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1699 struct netlink_callback *cb,
1700 int *q_idx_p, int s_q_idx, bool recur,
1701 bool dump_invisible)
1703 int ret = 0, q_idx = *q_idx_p;
1711 if (q_idx < s_q_idx) {
1714 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1715 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1716 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1722 /* If dumping singletons, there is no qdisc_dev(root) and the singleton
1723 * itself has already been dumped.
1725 * If we've already dumped the top-level (ingress) qdisc above and the global
1726 * qdisc hashtable, we don't want to hit it again
1728 if (!qdisc_dev(root) || !recur)
1731 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
1732 if (q_idx < s_q_idx) {
1736 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1737 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1738 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1752 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1754 struct net *net = sock_net(skb->sk);
1757 struct net_device *dev;
1758 const struct nlmsghdr *nlh = cb->nlh;
1759 struct nlattr *tca[TCA_MAX + 1];
1762 s_idx = cb->args[0];
1763 s_q_idx = q_idx = cb->args[1];
1768 err = nlmsg_parse_deprecated(nlh, sizeof(struct tcmsg), tca, TCA_MAX,
1769 rtm_tca_policy, cb->extack);
1773 for_each_netdev(net, dev) {
1774 struct netdev_queue *dev_queue;
1782 if (tc_dump_qdisc_root(rtnl_dereference(dev->qdisc),
1783 skb, cb, &q_idx, s_q_idx,
1784 true, tca[TCA_DUMP_INVISIBLE]) < 0)
1787 dev_queue = dev_ingress_queue(dev);
1789 tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
1790 &q_idx, s_q_idx, false,
1791 tca[TCA_DUMP_INVISIBLE]) < 0)
1800 cb->args[1] = q_idx;
1807 /************************************************
1808 * Traffic classes manipulation. *
1809 ************************************************/
1811 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1813 u32 portid, u32 seq, u16 flags, int event)
1816 struct nlmsghdr *nlh;
1817 unsigned char *b = skb_tail_pointer(skb);
1819 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1822 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1824 goto out_nlmsg_trim;
1825 tcm = nlmsg_data(nlh);
1826 tcm->tcm_family = AF_UNSPEC;
1829 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1830 tcm->tcm_parent = q->handle;
1831 tcm->tcm_handle = q->handle;
1833 if (nla_put_string(skb, TCA_KIND, q->ops->id))
1834 goto nla_put_failure;
1835 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1836 goto nla_put_failure;
1838 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1839 NULL, &d, TCA_PAD) < 0)
1840 goto nla_put_failure;
1842 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1843 goto nla_put_failure;
1845 if (gnet_stats_finish_copy(&d) < 0)
1846 goto nla_put_failure;
1848 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1857 static int tclass_notify(struct net *net, struct sk_buff *oskb,
1858 struct nlmsghdr *n, struct Qdisc *q,
1859 unsigned long cl, int event)
1861 struct sk_buff *skb;
1862 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1864 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1868 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event) < 0) {
1873 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1874 n->nlmsg_flags & NLM_F_ECHO);
1877 static int tclass_del_notify(struct net *net,
1878 const struct Qdisc_class_ops *cops,
1879 struct sk_buff *oskb, struct nlmsghdr *n,
1880 struct Qdisc *q, unsigned long cl,
1881 struct netlink_ext_ack *extack)
1883 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1884 struct sk_buff *skb;
1890 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1894 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,
1895 RTM_DELTCLASS) < 0) {
1900 err = cops->delete(q, cl, extack);
1906 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1907 n->nlmsg_flags & NLM_F_ECHO);
1911 #ifdef CONFIG_NET_CLS
1913 struct tcf_bind_args {
1914 struct tcf_walker w;
1920 static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
1922 struct tcf_bind_args *a = (void *)arg;
1924 if (n && tp->ops->bind_class) {
1925 struct Qdisc *q = tcf_block_q(tp->chain->block);
1928 tp->ops->bind_class(n, a->classid, a->cl, q, a->base);
1934 struct tc_bind_class_args {
1935 struct qdisc_walker w;
1936 unsigned long new_cl;
1941 static int tc_bind_class_walker(struct Qdisc *q, unsigned long cl,
1942 struct qdisc_walker *w)
1944 struct tc_bind_class_args *a = (struct tc_bind_class_args *)w;
1945 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1946 struct tcf_block *block;
1947 struct tcf_chain *chain;
1949 block = cops->tcf_block(q, cl, NULL);
1952 for (chain = tcf_get_next_chain(block, NULL);
1954 chain = tcf_get_next_chain(block, chain)) {
1955 struct tcf_proto *tp;
1957 for (tp = tcf_get_next_proto(chain, NULL);
1958 tp; tp = tcf_get_next_proto(chain, tp)) {
1959 struct tcf_bind_args arg = {};
1961 arg.w.fn = tcf_node_bind;
1962 arg.classid = a->clid;
1965 tp->ops->walk(tp, &arg.w, true);
1972 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
1973 unsigned long new_cl)
1975 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1976 struct tc_bind_class_args args = {};
1978 if (!cops->tcf_block)
1980 args.portid = portid;
1982 args.new_cl = new_cl;
1983 args.w.fn = tc_bind_class_walker;
1984 q->ops->cl_ops->walk(q, &args.w);
1989 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
1990 unsigned long new_cl)
1996 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n,
1997 struct netlink_ext_ack *extack)
1999 struct net *net = sock_net(skb->sk);
2000 struct tcmsg *tcm = nlmsg_data(n);
2001 struct nlattr *tca[TCA_MAX + 1];
2002 struct net_device *dev;
2003 struct Qdisc *q = NULL;
2004 const struct Qdisc_class_ops *cops;
2005 unsigned long cl = 0;
2006 unsigned long new_cl;
2012 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
2013 rtm_tca_policy, extack);
2017 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2022 parent == TC_H_UNSPEC - unspecified parent.
2023 parent == TC_H_ROOT - class is root, which has no parent.
2024 parent == X:0 - parent is root class.
2025 parent == X:Y - parent is a node in hierarchy.
2026 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
2028 handle == 0:0 - generate handle from kernel pool.
2029 handle == 0:Y - class is X:Y, where X:0 is qdisc.
2030 handle == X:Y - clear.
2031 handle == X:0 - root class.
2034 /* Step 1. Determine qdisc handle X:0 */
2036 portid = tcm->tcm_parent;
2037 clid = tcm->tcm_handle;
2038 qid = TC_H_MAJ(clid);
2040 if (portid != TC_H_ROOT) {
2041 u32 qid1 = TC_H_MAJ(portid);
2044 /* If both majors are known, they must be identical. */
2049 } else if (qid == 0)
2050 qid = rtnl_dereference(dev->qdisc)->handle;
2052 /* Now qid is genuine qdisc handle consistent
2053 * both with parent and child.
2055 * TC_H_MAJ(portid) still may be unspecified, complete it now.
2058 portid = TC_H_MAKE(qid, portid);
2061 qid = rtnl_dereference(dev->qdisc)->handle;
2064 /* OK. Locate qdisc */
2065 q = qdisc_lookup(dev, qid);
2069 /* An check that it supports classes */
2070 cops = q->ops->cl_ops;
2074 /* Now try to get class */
2076 if (portid == TC_H_ROOT)
2079 clid = TC_H_MAKE(qid, clid);
2082 cl = cops->find(q, clid);
2086 if (n->nlmsg_type != RTM_NEWTCLASS ||
2087 !(n->nlmsg_flags & NLM_F_CREATE))
2090 switch (n->nlmsg_type) {
2093 if (n->nlmsg_flags & NLM_F_EXCL)
2097 err = tclass_del_notify(net, cops, skb, n, q, cl, extack);
2098 /* Unbind the class with flilters with 0 */
2099 tc_bind_tclass(q, portid, clid, 0);
2102 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
2110 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
2111 NL_SET_ERR_MSG(extack, "Shared blocks are not supported for classes");
2118 err = cops->change(q, clid, portid, tca, &new_cl, extack);
2120 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
2121 /* We just create a new class, need to do reverse binding. */
2123 tc_bind_tclass(q, portid, clid, new_cl);
2129 struct qdisc_dump_args {
2130 struct qdisc_walker w;
2131 struct sk_buff *skb;
2132 struct netlink_callback *cb;
2135 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl,
2136 struct qdisc_walker *arg)
2138 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
2140 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid,
2141 a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2145 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
2146 struct tcmsg *tcm, struct netlink_callback *cb,
2149 struct qdisc_dump_args arg;
2151 if (tc_qdisc_dump_ignore(q, false) ||
2152 *t_p < s_t || !q->ops->cl_ops ||
2154 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
2159 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
2160 arg.w.fn = qdisc_class_dump;
2164 arg.w.skip = cb->args[1];
2166 q->ops->cl_ops->walk(q, &arg.w);
2167 cb->args[1] = arg.w.count;
2174 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
2175 struct tcmsg *tcm, struct netlink_callback *cb,
2176 int *t_p, int s_t, bool recur)
2184 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
2187 if (!qdisc_dev(root) || !recur)
2190 if (tcm->tcm_parent) {
2191 q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent));
2192 if (q && q != root &&
2193 tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2197 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
2198 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2205 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
2207 struct tcmsg *tcm = nlmsg_data(cb->nlh);
2208 struct net *net = sock_net(skb->sk);
2209 struct netdev_queue *dev_queue;
2210 struct net_device *dev;
2213 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2215 dev = dev_get_by_index(net, tcm->tcm_ifindex);
2222 if (tc_dump_tclass_root(rtnl_dereference(dev->qdisc),
2223 skb, tcm, cb, &t, s_t, true) < 0)
2226 dev_queue = dev_ingress_queue(dev);
2228 tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb,
2229 &t, s_t, false) < 0)
2239 #ifdef CONFIG_PROC_FS
2240 static int psched_show(struct seq_file *seq, void *v)
2242 seq_printf(seq, "%08x %08x %08x %08x\n",
2243 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
2245 (u32)NSEC_PER_SEC / hrtimer_resolution);
2250 static int __net_init psched_net_init(struct net *net)
2252 struct proc_dir_entry *e;
2254 e = proc_create_single("psched", 0, net->proc_net, psched_show);
2261 static void __net_exit psched_net_exit(struct net *net)
2263 remove_proc_entry("psched", net->proc_net);
2266 static int __net_init psched_net_init(struct net *net)
2271 static void __net_exit psched_net_exit(struct net *net)
2276 static struct pernet_operations psched_net_ops = {
2277 .init = psched_net_init,
2278 .exit = psched_net_exit,
2281 static int __init pktsched_init(void)
2285 err = register_pernet_subsys(&psched_net_ops);
2287 pr_err("pktsched_init: "
2288 "cannot initialize per netns operations\n");
2292 register_qdisc(&pfifo_fast_ops);
2293 register_qdisc(&pfifo_qdisc_ops);
2294 register_qdisc(&bfifo_qdisc_ops);
2295 register_qdisc(&pfifo_head_drop_qdisc_ops);
2296 register_qdisc(&mq_qdisc_ops);
2297 register_qdisc(&noqueue_qdisc_ops);
2299 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, 0);
2300 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, 0);
2301 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc,
2303 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, 0);
2304 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, 0);
2305 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass,
2311 subsys_initcall(pktsched_init);