net/sched/sch_api.c

   1 /*
   2  * net/sched/sch_api.c  Packet scheduler API.
   3  *
   4  *              This program is free software; you can redistribute it and/or
   5  *              modify it under the terms of the GNU General Public License
   6  *              as published by the Free Software Foundation; either version
   7  *              2 of the License, or (at your option) any later version.
   8  *
   9  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  10  *
  11  * Fixes:
  12  *
  13  * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
  14  * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
  15  * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
  16  */
  17
  18 #include <linux/module.h>
  19 #include <linux/types.h>
  20 #include <linux/kernel.h>
  21 #include <linux/string.h>
  22 #include <linux/errno.h>
  23 #include <linux/skbuff.h>
  24 #include <linux/init.h>
  25 #include <linux/proc_fs.h>
  26 #include <linux/seq_file.h>
  27 #include <linux/kmod.h>
  28 #include <linux/list.h>
  29 #include <linux/hrtimer.h>
  30 #include <linux/lockdep.h>
  31 #include <linux/slab.h>
  32
  33 #include <net/net_namespace.h>
  34 #include <net/sock.h>
  35 #include <net/netlink.h>
  36 #include <net/pkt_sched.h>
  37
  38 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
  39                         struct nlmsghdr *n, u32 clid,
  40                         struct Qdisc *old, struct Qdisc *new);
  41 static int tclass_notify(struct net *net, struct sk_buff *oskb,
  42                          struct nlmsghdr *n, struct Qdisc *q,
  43                          unsigned long cl, int event);
  44
  45 /*
  46
  47    Short review.
  48    -------------
  49
  50    This file consists of two interrelated parts:
  51
  52    1. queueing disciplines manager frontend.
  53    2. traffic classes manager frontend.
  54
  55    Generally, queueing discipline ("qdisc") is a black box,
  56    which is able to enqueue packets and to dequeue them (when
  57    device is ready to send something) in order and at times
  58    determined by algorithm hidden in it.
  59
  60    qdisc's are divided to two categories:
  61    - "queues", which have no internal structure visible from outside.
  62    - "schedulers", which split all the packets to "traffic classes",
  63      using "packet classifiers" (look at cls_api.c)
  64
  65    In turn, classes may have child qdiscs (as rule, queues)
  66    attached to them etc. etc. etc.
  67
  68    The goal of the routines in this file is to translate
  69    information supplied by user in the form of handles
  70    to more intelligible for kernel form, to make some sanity
  71    checks and part of work, which is common to all qdiscs
  72    and to provide rtnetlink notifications.
  73
  74    All real intelligent work is done inside qdisc modules.
  75
  76
  77
  78    Every discipline has two major routines: enqueue and dequeue.
  79
  80    ---dequeue
  81
  82    dequeue usually returns a skb to send. It is allowed to return NULL,
  83    but it does not mean that queue is empty, it just means that
  84    discipline does not want to send anything this time.
  85    Queue is really empty if q->q.qlen == 0.
  86    For complicated disciplines with multiple queues q->q is not
  87    real packet queue, but however q->q.qlen must be valid.
  88
  89    ---enqueue
  90
  91    enqueue returns 0, if packet was enqueued successfully.
  92    If packet (this one or another one) was dropped, it returns
  93    not zero error code.
  94    NET_XMIT_DROP        - this packet dropped
  95      Expected action: do not backoff, but wait until queue will clear.
  96    NET_XMIT_CN          - probably this packet enqueued, but another one dropped.
  97      Expected action: backoff or ignore
  98    NET_XMIT_POLICED     - dropped by police.
  99      Expected action: backoff or error to real-time apps.
 100
 101    Auxiliary routines:
 102
 103    ---peek
 104
 105    like dequeue but without removing a packet from the queue
 106
 107    ---reset
 108
 109    returns qdisc to initial state: purge all buffers, clear all
 110    timers, counters (except for statistics) etc.
 111
 112    ---init
 113
 114    initializes newly created qdisc.
 115
 116    ---destroy
 117
 118    destroys resources allocated by init and during lifetime of qdisc.
 119
 120    ---change
 121
 122    changes qdisc parameters.
 123  */
 124
 125 /* Protects list of registered TC modules. It is pure SMP lock. */
 126 static DEFINE_RWLOCK(qdisc_mod_lock);
 127
 128
 129 /************************************************
 130  *      Queueing disciplines manipulation.      *
 131  ************************************************/
 132
 133
 134 /* The list of all installed queueing disciplines. */
 135
 136 static struct Qdisc_ops *qdisc_base;
 137
 138 /* Register/unregister queueing discipline */
 139
 140 int register_qdisc(struct Qdisc_ops *qops)
 141 {
 142         struct Qdisc_ops *q, **qp;
 143         int rc = -EEXIST;
 144
 145         write_lock(&qdisc_mod_lock);
 146         for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
 147                 if (!strcmp(qops->id, q->id))
 148                         goto out;
 149
 150         if (qops->enqueue == NULL)
 151                 qops->enqueue = noop_qdisc_ops.enqueue;
 152         if (qops->peek == NULL) {
 153                 if (qops->dequeue == NULL)
 154                         qops->peek = noop_qdisc_ops.peek;
 155                 else
 156                         goto out_einval;
 157         }
 158         if (qops->dequeue == NULL)
 159                 qops->dequeue = noop_qdisc_ops.dequeue;
 160
 161         if (qops->cl_ops) {
 162                 const struct Qdisc_class_ops *cops = qops->cl_ops;
 163
 164                 if (!(cops->get && cops->put && cops->walk && cops->leaf))
 165                         goto out_einval;
 166
 167                 if (cops->tcf_chain && !(cops->bind_tcf && cops->unbind_tcf))
 168                         goto out_einval;
 169         }
 170
 171         qops->next = NULL;
 172         *qp = qops;
 173         rc = 0;
 174 out:
 175         write_unlock(&qdisc_mod_lock);
 176         return rc;
 177
 178 out_einval:
 179         rc = -EINVAL;
 180         goto out;
 181 }
 182 EXPORT_SYMBOL(register_qdisc);
 183
 184 int unregister_qdisc(struct Qdisc_ops *qops)
 185 {
 186         struct Qdisc_ops *q, **qp;
 187         int err = -ENOENT;
 188
 189         write_lock(&qdisc_mod_lock);
 190         for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
 191                 if (q == qops)
 192                         break;
 193         if (q) {
 194                 *qp = q->next;
 195                 q->next = NULL;
 196                 err = 0;
 197         }
 198         write_unlock(&qdisc_mod_lock);
 199         return err;
 200 }
 201 EXPORT_SYMBOL(unregister_qdisc);
 202
 203 /* Get default qdisc if not otherwise specified */
 204 void qdisc_get_default(char *name, size_t len)
 205 {
 206         read_lock(&qdisc_mod_lock);
 207         strlcpy(name, default_qdisc_ops->id, len);
 208         read_unlock(&qdisc_mod_lock);
 209 }
 210
 211 static struct Qdisc_ops *qdisc_lookup_default(const char *name)
 212 {
 213         struct Qdisc_ops *q = NULL;
 214
 215         for (q = qdisc_base; q; q = q->next) {
 216                 if (!strcmp(name, q->id)) {
 217                         if (!try_module_get(q->owner))
 218                                 q = NULL;
 219                         break;
 220                 }
 221         }
 222
 223         return q;
 224 }
 225
 226 /* Set new default qdisc to use */
 227 int qdisc_set_default(const char *name)
 228 {
 229         const struct Qdisc_ops *ops;
 230
 231         if (!capable(CAP_NET_ADMIN))
 232                 return -EPERM;
 233
 234         write_lock(&qdisc_mod_lock);
 235         ops = qdisc_lookup_default(name);
 236         if (!ops) {
 237                 /* Not found, drop lock and try to load module */
 238                 write_unlock(&qdisc_mod_lock);
 239                 request_module("sch_%s", name);
 240                 write_lock(&qdisc_mod_lock);
 241
 242                 ops = qdisc_lookup_default(name);
 243         }
 244
 245         if (ops) {
 246                 /* Set new default */
 247                 module_put(default_qdisc_ops->owner);
 248                 default_qdisc_ops = ops;
 249         }
 250         write_unlock(&qdisc_mod_lock);
 251
 252         return ops ? 0 : -ENOENT;
 253 }
 254
 255 /* We know handle. Find qdisc among all qdisc's attached to device
 256  * (root qdisc, all its children, children of children etc.)
 257  * Note: caller either uses rtnl or rcu_read_lock()
 258  */
 259
 260 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
 261 {
 262         struct Qdisc *q;
 263
 264         if (!(root->flags & TCQ_F_BUILTIN) &&
 265             root->handle == handle)
 266                 return root;
 267
 268         list_for_each_entry_rcu(q, &root->list, list) {
 269                 if (q->handle == handle)
 270                         return q;
 271         }
 272         return NULL;
 273 }
 274
 275 void qdisc_list_add(struct Qdisc *q)
 276 {
 277         if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
 278                 struct Qdisc *root = qdisc_dev(q)->qdisc;
 279
 280                 WARN_ON_ONCE(root == &noop_qdisc);
 281                 ASSERT_RTNL();
 282                 list_add_tail_rcu(&q->list, &root->list);
 283         }
 284 }
 285 EXPORT_SYMBOL(qdisc_list_add);
 286
 287 void qdisc_list_del(struct Qdisc *q)
 288 {
 289         if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
 290                 ASSERT_RTNL();
 291                 list_del_rcu(&q->list);
 292         }
 293 }
 294 EXPORT_SYMBOL(qdisc_list_del);
 295
 296 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
 297 {
 298         struct Qdisc *q;
 299
 300         q = qdisc_match_from_root(dev->qdisc, handle);
 301         if (q)
 302                 goto out;
 303
 304         if (dev_ingress_queue(dev))
 305                 q = qdisc_match_from_root(
 306                         dev_ingress_queue(dev)->qdisc_sleeping,
 307                         handle);
 308 out:
 309         return q;
 310 }
 311
 312 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
 313 {
 314         unsigned long cl;
 315         struct Qdisc *leaf;
 316         const struct Qdisc_class_ops *cops = p->ops->cl_ops;
 317
 318         if (cops == NULL)
 319                 return NULL;
 320         cl = cops->get(p, classid);
 321
 322         if (cl == 0)
 323                 return NULL;
 324         leaf = cops->leaf(p, cl);
 325         cops->put(p, cl);
 326         return leaf;
 327 }
 328
 329 /* Find queueing discipline by name */
 330
 331 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
 332 {
 333         struct Qdisc_ops *q = NULL;
 334
 335         if (kind) {
 336                 read_lock(&qdisc_mod_lock);
 337                 for (q = qdisc_base; q; q = q->next) {
 338                         if (nla_strcmp(kind, q->id) == 0) {
 339                                 if (!try_module_get(q->owner))
 340                                         q = NULL;
 341                                 break;
 342                         }
 343                 }
 344                 read_unlock(&qdisc_mod_lock);
 345         }
 346         return q;
 347 }
 348
 349 /* The linklayer setting were not transferred from iproute2, in older
 350  * versions, and the rate tables lookup systems have been dropped in
 351  * the kernel. To keep backward compatible with older iproute2 tc
 352  * utils, we detect the linklayer setting by detecting if the rate
 353  * table were modified.
 354  *
 355  * For linklayer ATM table entries, the rate table will be aligned to
 356  * 48 bytes, thus some table entries will contain the same value.  The
 357  * mpu (min packet unit) is also encoded into the old rate table, thus
 358  * starting from the mpu, we find low and high table entries for
 359  * mapping this cell.  If these entries contain the same value, when
 360  * the rate tables have been modified for linklayer ATM.
 361  *
 362  * This is done by rounding mpu to the nearest 48 bytes cell/entry,
 363  * and then roundup to the next cell, calc the table entry one below,
 364  * and compare.
 365  */
 366 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab)
 367 {
 368         int low       = roundup(r->mpu, 48);
 369         int high      = roundup(low+1, 48);
 370         int cell_low  = low >> r->cell_log;
 371         int cell_high = (high >> r->cell_log) - 1;
 372
 373         /* rtab is too inaccurate at rates > 100Mbit/s */
 374         if ((r->rate > (100000000/8)) || (rtab[0] == 0)) {
 375                 pr_debug("TC linklayer: Giving up ATM detection\n");
 376                 return TC_LINKLAYER_ETHERNET;
 377         }
 378
 379         if ((cell_high > cell_low) && (cell_high < 256)
 380             && (rtab[cell_low] == rtab[cell_high])) {
 381                 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n",
 382                          cell_low, cell_high, rtab[cell_high]);
 383                 return TC_LINKLAYER_ATM;
 384         }
 385         return TC_LINKLAYER_ETHERNET;
 386 }
 387
 388 static struct qdisc_rate_table *qdisc_rtab_list;
 389
 390 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab)
 391 {
 392         struct qdisc_rate_table *rtab;
 393
 394         if (tab == NULL || r->rate == 0 ||
 395             r->cell_log == 0 || r->cell_log >= 32 ||
 396             nla_len(tab) != TC_RTAB_SIZE)
 397                 return NULL;
 398
 399         for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
 400                 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) &&
 401                     !memcmp(&rtab->data, nla_data(tab), 1024)) {
 402                         rtab->refcnt++;
 403                         return rtab;
 404                 }
 405         }
 406
 407         rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
 408         if (rtab) {
 409                 rtab->rate = *r;
 410                 rtab->refcnt = 1;
 411                 memcpy(rtab->data, nla_data(tab), 1024);
 412                 if (r->linklayer == TC_LINKLAYER_UNAWARE)
 413                         r->linklayer = __detect_linklayer(r, rtab->data);
 414                 rtab->next = qdisc_rtab_list;
 415                 qdisc_rtab_list = rtab;
 416         }
 417         return rtab;
 418 }
 419 EXPORT_SYMBOL(qdisc_get_rtab);
 420
 421 void qdisc_put_rtab(struct qdisc_rate_table *tab)
 422 {
 423         struct qdisc_rate_table *rtab, **rtabp;
 424
 425         if (!tab || --tab->refcnt)
 426                 return;
 427
 428         for (rtabp = &qdisc_rtab_list;
 429              (rtab = *rtabp) != NULL;
 430              rtabp = &rtab->next) {
 431                 if (rtab == tab) {
 432                         *rtabp = rtab->next;
 433                         kfree(rtab);
 434                         return;
 435                 }
 436         }
 437 }
 438 EXPORT_SYMBOL(qdisc_put_rtab);
 439
 440 static LIST_HEAD(qdisc_stab_list);
 441 static DEFINE_SPINLOCK(qdisc_stab_lock);
 442
 443 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
 444         [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
 445         [TCA_STAB_DATA] = { .type = NLA_BINARY },
 446 };
 447
 448 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt)
 449 {
 450         struct nlattr *tb[TCA_STAB_MAX + 1];
 451         struct qdisc_size_table *stab;
 452         struct tc_sizespec *s;
 453         unsigned int tsize = 0;
 454         u16 *tab = NULL;
 455         int err;
 456
 457         err = nla_parse_nested(tb, TCA_STAB_MAX, opt, stab_policy);
 458         if (err < 0)
 459                 return ERR_PTR(err);
 460         if (!tb[TCA_STAB_BASE])
 461                 return ERR_PTR(-EINVAL);
 462
 463         s = nla_data(tb[TCA_STAB_BASE]);
 464
 465         if (s->tsize > 0) {
 466                 if (!tb[TCA_STAB_DATA])
 467                         return ERR_PTR(-EINVAL);
 468                 tab = nla_data(tb[TCA_STAB_DATA]);
 469                 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
 470         }
 471
 472         if (tsize != s->tsize || (!tab && tsize > 0))
 473                 return ERR_PTR(-EINVAL);
 474
 475         spin_lock(&qdisc_stab_lock);
 476
 477         list_for_each_entry(stab, &qdisc_stab_list, list) {
 478                 if (memcmp(&stab->szopts, s, sizeof(*s)))
 479                         continue;
 480                 if (tsize > 0 && memcmp(stab->data, tab, tsize * sizeof(u16)))
 481                         continue;
 482                 stab->refcnt++;
 483                 spin_unlock(&qdisc_stab_lock);
 484                 return stab;
 485         }
 486
 487         spin_unlock(&qdisc_stab_lock);
 488
 489         stab = kmalloc(sizeof(*stab) + tsize * sizeof(u16), GFP_KERNEL);
 490         if (!stab)
 491                 return ERR_PTR(-ENOMEM);
 492
 493         stab->refcnt = 1;
 494         stab->szopts = *s;
 495         if (tsize > 0)
 496                 memcpy(stab->data, tab, tsize * sizeof(u16));
 497
 498         spin_lock(&qdisc_stab_lock);
 499         list_add_tail(&stab->list, &qdisc_stab_list);
 500         spin_unlock(&qdisc_stab_lock);
 501
 502         return stab;
 503 }
 504
 505 static void stab_kfree_rcu(struct rcu_head *head)
 506 {
 507         kfree(container_of(head, struct qdisc_size_table, rcu));
 508 }
 509
 510 void qdisc_put_stab(struct qdisc_size_table *tab)
 511 {
 512         if (!tab)
 513                 return;
 514
 515         spin_lock(&qdisc_stab_lock);
 516
 517         if (--tab->refcnt == 0) {
 518                 list_del(&tab->list);
 519                 call_rcu_bh(&tab->rcu, stab_kfree_rcu);
 520         }
 521
 522         spin_unlock(&qdisc_stab_lock);
 523 }
 524 EXPORT_SYMBOL(qdisc_put_stab);
 525
 526 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
 527 {
 528         struct nlattr *nest;
 529
 530         nest = nla_nest_start(skb, TCA_STAB);
 531         if (nest == NULL)
 532                 goto nla_put_failure;
 533         if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts))
 534                 goto nla_put_failure;
 535         nla_nest_end(skb, nest);
 536
 537         return skb->len;
 538
 539 nla_put_failure:
 540         return -1;
 541 }
 542
 543 void __qdisc_calculate_pkt_len(struct sk_buff *skb, const struct qdisc_size_table *stab)
 544 {
 545         int pkt_len, slot;
 546
 547         pkt_len = skb->len + stab->szopts.overhead;
 548         if (unlikely(!stab->szopts.tsize))
 549                 goto out;
 550
 551         slot = pkt_len + stab->szopts.cell_align;
 552         if (unlikely(slot < 0))
 553                 slot = 0;
 554
 555         slot >>= stab->szopts.cell_log;
 556         if (likely(slot < stab->szopts.tsize))
 557                 pkt_len = stab->data[slot];
 558         else
 559                 pkt_len = stab->data[stab->szopts.tsize - 1] *
 560                                 (slot / stab->szopts.tsize) +
 561                                 stab->data[slot % stab->szopts.tsize];
 562
 563         pkt_len <<= stab->szopts.size_log;
 564 out:
 565         if (unlikely(pkt_len < 1))
 566                 pkt_len = 1;
 567         qdisc_skb_cb(skb)->pkt_len = pkt_len;
 568 }
 569 EXPORT_SYMBOL(__qdisc_calculate_pkt_len);
 570
 571 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc)
 572 {
 573         if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
 574                 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n",
 575                         txt, qdisc->ops->id, qdisc->handle >> 16);
 576                 qdisc->flags |= TCQ_F_WARN_NONWC;
 577         }
 578 }
 579 EXPORT_SYMBOL(qdisc_warn_nonwc);
 580
 581 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
 582 {
 583         struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
 584                                                  timer);
 585
 586         rcu_read_lock();
 587         qdisc_unthrottled(wd->qdisc);
 588         __netif_schedule(qdisc_root(wd->qdisc));
 589         rcu_read_unlock();
 590
 591         return HRTIMER_NORESTART;
 592 }
 593
 594 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
 595 {
 596         hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
 597         wd->timer.function = qdisc_watchdog;
 598         wd->qdisc = qdisc;
 599 }
 600 EXPORT_SYMBOL(qdisc_watchdog_init);
 601
 602 void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires, bool throttle)
 603 {
 604         if (test_bit(__QDISC_STATE_DEACTIVATED,
 605                      &qdisc_root_sleeping(wd->qdisc)->state))
 606                 return;
 607
 608         if (throttle)
 609                 qdisc_throttled(wd->qdisc);
 610
 611         hrtimer_start(&wd->timer,
 612                       ns_to_ktime(expires),
 613                       HRTIMER_MODE_ABS_PINNED);
 614 }
 615 EXPORT_SYMBOL(qdisc_watchdog_schedule_ns);
 616
 617 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
 618 {
 619         hrtimer_cancel(&wd->timer);
 620         qdisc_unthrottled(wd->qdisc);
 621 }
 622 EXPORT_SYMBOL(qdisc_watchdog_cancel);
 623
 624 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
 625 {
 626         unsigned int size = n * sizeof(struct hlist_head), i;
 627         struct hlist_head *h;
 628
 629         if (size <= PAGE_SIZE)
 630                 h = kmalloc(size, GFP_KERNEL);
 631         else
 632                 h = (struct hlist_head *)
 633                         __get_free_pages(GFP_KERNEL, get_order(size));
 634
 635         if (h != NULL) {
 636                 for (i = 0; i < n; i++)
 637                         INIT_HLIST_HEAD(&h[i]);
 638         }
 639         return h;
 640 }
 641
 642 static void qdisc_class_hash_free(struct hlist_head *h, unsigned int n)
 643 {
 644         unsigned int size = n * sizeof(struct hlist_head);
 645
 646         if (size <= PAGE_SIZE)
 647                 kfree(h);
 648         else
 649                 free_pages((unsigned long)h, get_order(size));
 650 }
 651
 652 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
 653 {
 654         struct Qdisc_class_common *cl;
 655         struct hlist_node *next;
 656         struct hlist_head *nhash, *ohash;
 657         unsigned int nsize, nmask, osize;
 658         unsigned int i, h;
 659
 660         /* Rehash when load factor exceeds 0.75 */
 661         if (clhash->hashelems * 4 <= clhash->hashsize * 3)
 662                 return;
 663         nsize = clhash->hashsize * 2;
 664         nmask = nsize - 1;
 665         nhash = qdisc_class_hash_alloc(nsize);
 666         if (nhash == NULL)
 667                 return;
 668
 669         ohash = clhash->hash;
 670         osize = clhash->hashsize;
 671
 672         sch_tree_lock(sch);
 673         for (i = 0; i < osize; i++) {
 674                 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) {
 675                         h = qdisc_class_hash(cl->classid, nmask);
 676                         hlist_add_head(&cl->hnode, &nhash[h]);
 677                 }
 678         }
 679         clhash->hash     = nhash;
 680         clhash->hashsize = nsize;
 681         clhash->hashmask = nmask;
 682         sch_tree_unlock(sch);
 683
 684         qdisc_class_hash_free(ohash, osize);
 685 }
 686 EXPORT_SYMBOL(qdisc_class_hash_grow);
 687
 688 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
 689 {
 690         unsigned int size = 4;
 691
 692         clhash->hash = qdisc_class_hash_alloc(size);
 693         if (clhash->hash == NULL)
 694                 return -ENOMEM;
 695         clhash->hashsize  = size;
 696         clhash->hashmask  = size - 1;
 697         clhash->hashelems = 0;
 698         return 0;
 699 }
 700 EXPORT_SYMBOL(qdisc_class_hash_init);
 701
 702 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
 703 {
 704         qdisc_class_hash_free(clhash->hash, clhash->hashsize);
 705 }
 706 EXPORT_SYMBOL(qdisc_class_hash_destroy);
 707
 708 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
 709                              struct Qdisc_class_common *cl)
 710 {
 711         unsigned int h;
 712
 713         INIT_HLIST_NODE(&cl->hnode);
 714         h = qdisc_class_hash(cl->classid, clhash->hashmask);
 715         hlist_add_head(&cl->hnode, &clhash->hash[h]);
 716         clhash->hashelems++;
 717 }
 718 EXPORT_SYMBOL(qdisc_class_hash_insert);
 719
 720 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
 721                              struct Qdisc_class_common *cl)
 722 {
 723         hlist_del(&cl->hnode);
 724         clhash->hashelems--;
 725 }
 726 EXPORT_SYMBOL(qdisc_class_hash_remove);
 727
 728 /* Allocate an unique handle from space managed by kernel
 729  * Possible range is [8000-FFFF]:0000 (0x8000 values)
 730  */
 731 static u32 qdisc_alloc_handle(struct net_device *dev)
 732 {
 733         int i = 0x8000;
 734         static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
 735
 736         do {
 737                 autohandle += TC_H_MAKE(0x10000U, 0);
 738                 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
 739                         autohandle = TC_H_MAKE(0x80000000U, 0);
 740                 if (!qdisc_lookup(dev, autohandle))
 741                         return autohandle;
 742                 cond_resched();
 743         } while (--i > 0);
 744
 745         return 0;
 746 }
 747
 748 void qdisc_tree_reduce_backlog(struct Qdisc *sch, unsigned int n,
 749                                unsigned int len)
 750 {
 751         const struct Qdisc_class_ops *cops;
 752         unsigned long cl;
 753         u32 parentid;
 754         int drops;
 755
 756         if (n == 0 && len == 0)
 757                 return;
 758         drops = max_t(int, n, 0);
 759         rcu_read_lock();
 760         while ((parentid = sch->parent)) {
 761                 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
 762                         break;
 763
 764                 if (sch->flags & TCQ_F_NOPARENT)
 765                         break;
 766                 /* TODO: perform the search on a per txq basis */
 767                 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
 768                 if (sch == NULL) {
 769                         WARN_ON_ONCE(parentid != TC_H_ROOT);
 770                         break;
 771                 }
 772                 cops = sch->ops->cl_ops;
 773                 if (cops->qlen_notify) {
 774                         cl = cops->get(sch, parentid);
 775                         cops->qlen_notify(sch, cl);
 776                         cops->put(sch, cl);
 777                 }
 778                 sch->q.qlen -= n;
 779                 sch->qstats.backlog -= len;
 780                 __qdisc_qstats_drop(sch, drops);
 781         }
 782         rcu_read_unlock();
 783 }
 784 EXPORT_SYMBOL(qdisc_tree_reduce_backlog);
 785
 786 static void notify_and_destroy(struct net *net, struct sk_buff *skb,
 787                                struct nlmsghdr *n, u32 clid,
 788                                struct Qdisc *old, struct Qdisc *new)
 789 {
 790         if (new || old)
 791                 qdisc_notify(net, skb, n, clid, old, new);
 792
 793         if (old)
 794                 qdisc_destroy(old);
 795 }
 796
 797 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
 798  * to device "dev".
 799  *
 800  * When appropriate send a netlink notification using 'skb'
 801  * and "n".
 802  *
 803  * On success, destroy old qdisc.
 804  */
 805
 806 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
 807                        struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
 808                        struct Qdisc *new, struct Qdisc *old)
 809 {
 810         struct Qdisc *q = old;
 811         struct net *net = dev_net(dev);
 812         int err = 0;
 813
 814         if (parent == NULL) {
 815                 unsigned int i, num_q, ingress;
 816
 817                 ingress = 0;
 818                 num_q = dev->num_tx_queues;
 819                 if ((q && q->flags & TCQ_F_INGRESS) ||
 820                     (new && new->flags & TCQ_F_INGRESS)) {
 821                         num_q = 1;
 822                         ingress = 1;
 823                         if (!dev_ingress_queue(dev))
 824                                 return -ENOENT;
 825                 }
 826
 827                 if (dev->flags & IFF_UP)
 828                         dev_deactivate(dev);
 829
 830                 if (new && new->ops->attach)
 831                         goto skip;
 832
 833                 for (i = 0; i < num_q; i++) {
 834                         struct netdev_queue *dev_queue = dev_ingress_queue(dev);
 835
 836                         if (!ingress)
 837                                 dev_queue = netdev_get_tx_queue(dev, i);
 838
 839                         old = dev_graft_qdisc(dev_queue, new);
 840                         if (new && i > 0)
 841                                 atomic_inc(&new->refcnt);
 842
 843                         if (!ingress)
 844                                 qdisc_destroy(old);
 845                 }
 846
 847 skip:
 848                 if (!ingress) {
 849                         notify_and_destroy(net, skb, n, classid,
 850                                            dev->qdisc, new);
 851                         if (new && !new->ops->attach)
 852                                 atomic_inc(&new->refcnt);
 853                         dev->qdisc = new ? : &noop_qdisc;
 854
 855                         if (new && new->ops->attach)
 856                                 new->ops->attach(new);
 857                 } else {
 858                         notify_and_destroy(net, skb, n, classid, old, new);
 859                 }
 860
 861                 if (dev->flags & IFF_UP)
 862                         dev_activate(dev);
 863         } else {
 864                 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
 865
 866                 err = -EOPNOTSUPP;
 867                 if (cops && cops->graft) {
 868                         unsigned long cl = cops->get(parent, classid);
 869                         if (cl) {
 870                                 err = cops->graft(parent, cl, new, &old);
 871                                 cops->put(parent, cl);
 872                         } else
 873                                 err = -ENOENT;
 874                 }
 875                 if (!err)
 876                         notify_and_destroy(net, skb, n, classid, old, new);
 877         }
 878         return err;
 879 }
 880
 881 /* lockdep annotation is needed for ingress; egress gets it only for name */
 882 static struct lock_class_key qdisc_tx_lock;
 883 static struct lock_class_key qdisc_rx_lock;
 884
 885 /*
 886    Allocate and initialize new qdisc.
 887
 888    Parameters are passed via opt.
 889  */
 890
 891 static struct Qdisc *
 892 qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
 893              struct Qdisc *p, u32 parent, u32 handle,
 894              struct nlattr **tca, int *errp)
 895 {
 896         int err;
 897         struct nlattr *kind = tca[TCA_KIND];
 898         struct Qdisc *sch;
 899         struct Qdisc_ops *ops;
 900         struct qdisc_size_table *stab;
 901
 902         ops = qdisc_lookup_ops(kind);
 903 #ifdef CONFIG_MODULES
 904         if (ops == NULL && kind != NULL) {
 905                 char name[IFNAMSIZ];
 906                 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
 907                         /* We dropped the RTNL semaphore in order to
 908                          * perform the module load.  So, even if we
 909                          * succeeded in loading the module we have to
 910                          * tell the caller to replay the request.  We
 911                          * indicate this using -EAGAIN.
 912                          * We replay the request because the device may
 913                          * go away in the mean time.
 914                          */
 915                         rtnl_unlock();
 916                         request_module("sch_%s", name);
 917                         rtnl_lock();
 918                         ops = qdisc_lookup_ops(kind);
 919                         if (ops != NULL) {
 920                                 /* We will try again qdisc_lookup_ops,
 921                                  * so don't keep a reference.
 922                                  */
 923                                 module_put(ops->owner);
 924                                 err = -EAGAIN;
 925                                 goto err_out;
 926                         }
 927                 }
 928         }
 929 #endif
 930
 931         err = -ENOENT;
 932         if (ops == NULL)
 933                 goto err_out;
 934
 935         sch = qdisc_alloc(dev_queue, ops);
 936         if (IS_ERR(sch)) {
 937                 err = PTR_ERR(sch);
 938                 goto err_out2;
 939         }
 940
 941         sch->parent = parent;
 942
 943         if (handle == TC_H_INGRESS) {
 944                 sch->flags |= TCQ_F_INGRESS;
 945                 handle = TC_H_MAKE(TC_H_INGRESS, 0);
 946                 lockdep_set_class(qdisc_lock(sch), &qdisc_rx_lock);
 947         } else {
 948                 if (handle == 0) {
 949                         handle = qdisc_alloc_handle(dev);
 950                         err = -ENOMEM;
 951                         if (handle == 0)
 952                                 goto err_out3;
 953                 }
 954                 lockdep_set_class(qdisc_lock(sch), &qdisc_tx_lock);
 955                 if (!netif_is_multiqueue(dev))
 956                         sch->flags |= TCQ_F_ONETXQUEUE;
 957         }
 958
 959         sch->handle = handle;
 960
 961         if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
 962                 if (qdisc_is_percpu_stats(sch)) {
 963                         sch->cpu_bstats =
 964                                 netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu);
 965                         if (!sch->cpu_bstats)
 966                                 goto err_out4;
 967
 968                         sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
 969                         if (!sch->cpu_qstats)
 970                                 goto err_out4;
 971                 }
 972
 973                 if (tca[TCA_STAB]) {
 974                         stab = qdisc_get_stab(tca[TCA_STAB]);
 975                         if (IS_ERR(stab)) {
 976                                 err = PTR_ERR(stab);
 977                                 goto err_out4;
 978                         }
 979                         rcu_assign_pointer(sch->stab, stab);
 980                 }
 981                 if (tca[TCA_RATE]) {
 982                         spinlock_t *root_lock;
 983
 984                         err = -EOPNOTSUPP;
 985                         if (sch->flags & TCQ_F_MQROOT)
 986                                 goto err_out4;
 987
 988                         if ((sch->parent != TC_H_ROOT) &&
 989                             !(sch->flags & TCQ_F_INGRESS) &&
 990                             (!p || !(p->flags & TCQ_F_MQROOT)))
 991                                 root_lock = qdisc_root_sleeping_lock(sch);
 992                         else
 993                                 root_lock = qdisc_lock(sch);
 994
 995                         err = gen_new_estimator(&sch->bstats,
 996                                                 sch->cpu_bstats,
 997                                                 &sch->rate_est,
 998                                                 root_lock,
 999                                                 tca[TCA_RATE]);
1000                         if (err)
1001                                 goto err_out4;
1002                 }
1003
1004                 qdisc_list_add(sch);
1005
1006                 return sch;
1007         }
1008         /* ops->init() failed, we call ->destroy() like qdisc_create_dflt() */
1009         if (ops->destroy)
1010                 ops->destroy(sch);
1011 err_out3:
1012         dev_put(dev);
1013         kfree((char *) sch - sch->padded);
1014 err_out2:
1015         module_put(ops->owner);
1016 err_out:
1017         *errp = err;
1018         return NULL;
1019
1020 err_out4:
1021         free_percpu(sch->cpu_bstats);
1022         free_percpu(sch->cpu_qstats);
1023         /*
1024          * Any broken qdiscs that would require a ops->reset() here?
1025          * The qdisc was never in action so it shouldn't be necessary.
1026          */
1027         qdisc_put_stab(rtnl_dereference(sch->stab));
1028         if (ops->destroy)
1029                 ops->destroy(sch);
1030         goto err_out3;
1031 }
1032
1033 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
1034 {
1035         struct qdisc_size_table *ostab, *stab = NULL;
1036         int err = 0;
1037
1038         if (tca[TCA_OPTIONS]) {
1039                 if (sch->ops->change == NULL)
1040                         return -EINVAL;
1041                 err = sch->ops->change(sch, tca[TCA_OPTIONS]);
1042                 if (err)
1043                         return err;
1044         }
1045
1046         if (tca[TCA_STAB]) {
1047                 stab = qdisc_get_stab(tca[TCA_STAB]);
1048                 if (IS_ERR(stab))
1049                         return PTR_ERR(stab);
1050         }
1051
1052         ostab = rtnl_dereference(sch->stab);
1053         rcu_assign_pointer(sch->stab, stab);
1054         qdisc_put_stab(ostab);
1055
1056         if (tca[TCA_RATE]) {
1057                 /* NB: ignores errors from replace_estimator
1058                    because change can't be undone. */
1059                 if (sch->flags & TCQ_F_MQROOT)
1060                         goto out;
1061                 gen_replace_estimator(&sch->bstats,
1062                                       sch->cpu_bstats,
1063                                       &sch->rate_est,
1064                                       qdisc_root_sleeping_lock(sch),
1065                                       tca[TCA_RATE]);
1066         }
1067 out:
1068         return 0;
1069 }
1070
1071 struct check_loop_arg {
1072         struct qdisc_walker     w;
1073         struct Qdisc            *p;
1074         int                     depth;
1075 };
1076
1077 static int check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w);
1078
1079 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
1080 {
1081         struct check_loop_arg   arg;
1082
1083         if (q->ops->cl_ops == NULL)
1084                 return 0;
1085
1086         arg.w.stop = arg.w.skip = arg.w.count = 0;
1087         arg.w.fn = check_loop_fn;
1088         arg.depth = depth;
1089         arg.p = p;
1090         q->ops->cl_ops->walk(q, &arg.w);
1091         return arg.w.stop ? -ELOOP : 0;
1092 }
1093
1094 static int
1095 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
1096 {
1097         struct Qdisc *leaf;
1098         const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1099         struct check_loop_arg *arg = (struct check_loop_arg *)w;
1100
1101         leaf = cops->leaf(q, cl);
1102         if (leaf) {
1103                 if (leaf == arg->p || arg->depth > 7)
1104                         return -ELOOP;
1105                 return check_loop(leaf, arg->p, arg->depth + 1);
1106         }
1107         return 0;
1108 }
1109
1110 /*
1111  * Delete/get qdisc.
1112  */
1113
1114 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n)
1115 {
1116         struct net *net = sock_net(skb->sk);
1117         struct tcmsg *tcm = nlmsg_data(n);
1118         struct nlattr *tca[TCA_MAX + 1];
1119         struct net_device *dev;
1120         u32 clid;
1121         struct Qdisc *q = NULL;
1122         struct Qdisc *p = NULL;
1123         int err;
1124
1125         if ((n->nlmsg_type != RTM_GETQDISC) &&
1126             !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1127                 return -EPERM;
1128
1129         err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1130         if (err < 0)
1131                 return err;
1132
1133         dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1134         if (!dev)
1135                 return -ENODEV;
1136
1137         clid = tcm->tcm_parent;
1138         if (clid) {
1139                 if (clid != TC_H_ROOT) {
1140                         if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
1141                                 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1142                                 if (!p)
1143                                         return -ENOENT;
1144                                 q = qdisc_leaf(p, clid);
1145                         } else if (dev_ingress_queue(dev)) {
1146                                 q = dev_ingress_queue(dev)->qdisc_sleeping;
1147                         }
1148                 } else {
1149                         q = dev->qdisc;
1150                 }
1151                 if (!q)
1152                         return -ENOENT;
1153
1154                 if (tcm->tcm_handle && q->handle != tcm->tcm_handle)
1155                         return -EINVAL;
1156         } else {
1157                 q = qdisc_lookup(dev, tcm->tcm_handle);
1158                 if (!q)
1159                         return -ENOENT;
1160         }
1161
1162         if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1163                 return -EINVAL;
1164
1165         if (n->nlmsg_type == RTM_DELQDISC) {
1166                 if (!clid)
1167                         return -EINVAL;
1168                 if (q->handle == 0)
1169                         return -ENOENT;
1170                 err = qdisc_graft(dev, p, skb, n, clid, NULL, q);
1171                 if (err != 0)
1172                         return err;
1173         } else {
1174                 qdisc_notify(net, skb, n, clid, NULL, q);
1175         }
1176         return 0;
1177 }
1178
1179 /*
1180  * Create/change qdisc.
1181  */
1182
1183 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n)
1184 {
1185         struct net *net = sock_net(skb->sk);
1186         struct tcmsg *tcm;
1187         struct nlattr *tca[TCA_MAX + 1];
1188         struct net_device *dev;
1189         u32 clid;
1190         struct Qdisc *q, *p;
1191         int err;
1192
1193         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1194                 return -EPERM;
1195
1196 replay:
1197         /* Reinit, just in case something touches this. */
1198         err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1199         if (err < 0)
1200                 return err;
1201
1202         tcm = nlmsg_data(n);
1203         clid = tcm->tcm_parent;
1204         q = p = NULL;
1205
1206         dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1207         if (!dev)
1208                 return -ENODEV;
1209
1210
1211         if (clid) {
1212                 if (clid != TC_H_ROOT) {
1213                         if (clid != TC_H_INGRESS) {
1214                                 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1215                                 if (!p)
1216                                         return -ENOENT;
1217                                 q = qdisc_leaf(p, clid);
1218                         } else if (dev_ingress_queue_create(dev)) {
1219                                 q = dev_ingress_queue(dev)->qdisc_sleeping;
1220                         }
1221                 } else {
1222                         q = dev->qdisc;
1223                 }
1224
1225                 /* It may be default qdisc, ignore it */
1226                 if (q && q->handle == 0)
1227                         q = NULL;
1228
1229                 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1230                         if (tcm->tcm_handle) {
1231                                 if (q && !(n->nlmsg_flags & NLM_F_REPLACE))
1232                                         return -EEXIST;
1233                                 if (TC_H_MIN(tcm->tcm_handle))
1234                                         return -EINVAL;
1235                                 q = qdisc_lookup(dev, tcm->tcm_handle);
1236                                 if (!q)
1237                                         goto create_n_graft;
1238                                 if (n->nlmsg_flags & NLM_F_EXCL)
1239                                         return -EEXIST;
1240                                 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1241                                         return -EINVAL;
1242                                 if (q == p ||
1243                                     (p && check_loop(q, p, 0)))
1244                                         return -ELOOP;
1245                                 atomic_inc(&q->refcnt);
1246                                 goto graft;
1247                         } else {
1248                                 if (!q)
1249                                         goto create_n_graft;
1250
1251                                 /* This magic test requires explanation.
1252                                  *
1253                                  *   We know, that some child q is already
1254                                  *   attached to this parent and have choice:
1255                                  *   either to change it or to create/graft new one.
1256                                  *
1257                                  *   1. We are allowed to create/graft only
1258                                  *   if CREATE and REPLACE flags are set.
1259                                  *
1260                                  *   2. If EXCL is set, requestor wanted to say,
1261                                  *   that qdisc tcm_handle is not expected
1262                                  *   to exist, so that we choose create/graft too.
1263                                  *
1264                                  *   3. The last case is when no flags are set.
1265                                  *   Alas, it is sort of hole in API, we
1266                                  *   cannot decide what to do unambiguously.
1267                                  *   For now we select create/graft, if
1268                                  *   user gave KIND, which does not match existing.
1269                                  */
1270                                 if ((n->nlmsg_flags & NLM_F_CREATE) &&
1271                                     (n->nlmsg_flags & NLM_F_REPLACE) &&
1272                                     ((n->nlmsg_flags & NLM_F_EXCL) ||
1273                                      (tca[TCA_KIND] &&
1274                                       nla_strcmp(tca[TCA_KIND], q->ops->id))))
1275                                         goto create_n_graft;
1276                         }
1277                 }
1278         } else {
1279                 if (!tcm->tcm_handle)
1280                         return -EINVAL;
1281                 q = qdisc_lookup(dev, tcm->tcm_handle);
1282         }
1283
1284         /* Change qdisc parameters */
1285         if (q == NULL)
1286                 return -ENOENT;
1287         if (n->nlmsg_flags & NLM_F_EXCL)
1288                 return -EEXIST;
1289         if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1290                 return -EINVAL;
1291         err = qdisc_change(q, tca);
1292         if (err == 0)
1293                 qdisc_notify(net, skb, n, clid, NULL, q);
1294         return err;
1295
1296 create_n_graft:
1297         if (!(n->nlmsg_flags & NLM_F_CREATE))
1298                 return -ENOENT;
1299         if (clid == TC_H_INGRESS) {
1300                 if (dev_ingress_queue(dev))
1301                         q = qdisc_create(dev, dev_ingress_queue(dev), p,
1302                                          tcm->tcm_parent, tcm->tcm_parent,
1303                                          tca, &err);
1304                 else
1305                         err = -ENOENT;
1306         } else {
1307                 struct netdev_queue *dev_queue;
1308
1309                 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
1310                         dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1311                 else if (p)
1312                         dev_queue = p->dev_queue;
1313                 else
1314                         dev_queue = netdev_get_tx_queue(dev, 0);
1315
1316                 q = qdisc_create(dev, dev_queue, p,
1317                                  tcm->tcm_parent, tcm->tcm_handle,
1318                                  tca, &err);
1319         }
1320         if (q == NULL) {
1321                 if (err == -EAGAIN)
1322                         goto replay;
1323                 return err;
1324         }
1325
1326 graft:
1327         err = qdisc_graft(dev, p, skb, n, clid, q, NULL);
1328         if (err) {
1329                 if (q)
1330                         qdisc_destroy(q);
1331                 return err;
1332         }
1333
1334         return 0;
1335 }
1336
1337 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
1338                          u32 portid, u32 seq, u16 flags, int event)
1339 {
1340         struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL;
1341         struct gnet_stats_queue __percpu *cpu_qstats = NULL;
1342         struct tcmsg *tcm;
1343         struct nlmsghdr  *nlh;
1344         unsigned char *b = skb_tail_pointer(skb);
1345         struct gnet_dump d;
1346         struct qdisc_size_table *stab;
1347         __u32 qlen;
1348
1349         cond_resched();
1350         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1351         if (!nlh)
1352                 goto out_nlmsg_trim;
1353         tcm = nlmsg_data(nlh);
1354         tcm->tcm_family = AF_UNSPEC;
1355         tcm->tcm__pad1 = 0;
1356         tcm->tcm__pad2 = 0;
1357         tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1358         tcm->tcm_parent = clid;
1359         tcm->tcm_handle = q->handle;
1360         tcm->tcm_info = atomic_read(&q->refcnt);
1361         if (nla_put_string(skb, TCA_KIND, q->ops->id))
1362                 goto nla_put_failure;
1363         if (q->ops->dump && q->ops->dump(q, skb) < 0)
1364                 goto nla_put_failure;
1365         qlen = q->q.qlen;
1366
1367         stab = rtnl_dereference(q->stab);
1368         if (stab && qdisc_dump_stab(skb, stab) < 0)
1369                 goto nla_put_failure;
1370
1371         if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1372                                          qdisc_root_sleeping_lock(q), &d) < 0)
1373                 goto nla_put_failure;
1374
1375         if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
1376                 goto nla_put_failure;
1377
1378         if (qdisc_is_percpu_stats(q)) {
1379                 cpu_bstats = q->cpu_bstats;
1380                 cpu_qstats = q->cpu_qstats;
1381         }
1382
1383         if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats) < 0 ||
1384             gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 ||
1385             gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
1386                 goto nla_put_failure;
1387
1388         if (gnet_stats_finish_copy(&d) < 0)
1389                 goto nla_put_failure;
1390
1391         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1392         return skb->len;
1393
1394 out_nlmsg_trim:
1395 nla_put_failure:
1396         nlmsg_trim(skb, b);
1397         return -1;
1398 }
1399
1400 static bool tc_qdisc_dump_ignore(struct Qdisc *q)
1401 {
1402         return (q->flags & TCQ_F_BUILTIN) ? true : false;
1403 }
1404
1405 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
1406                         struct nlmsghdr *n, u32 clid,
1407                         struct Qdisc *old, struct Qdisc *new)
1408 {
1409         struct sk_buff *skb;
1410         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1411
1412         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1413         if (!skb)
1414                 return -ENOBUFS;
1415
1416         if (old && !tc_qdisc_dump_ignore(old)) {
1417                 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
1418                                   0, RTM_DELQDISC) < 0)
1419                         goto err_out;
1420         }
1421         if (new && !tc_qdisc_dump_ignore(new)) {
1422                 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
1423                                   old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
1424                         goto err_out;
1425         }
1426
1427         if (skb->len)
1428                 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1429                                       n->nlmsg_flags & NLM_F_ECHO);
1430
1431 err_out:
1432         kfree_skb(skb);
1433         return -EINVAL;
1434 }
1435
1436 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1437                               struct netlink_callback *cb,
1438                               int *q_idx_p, int s_q_idx)
1439 {
1440         int ret = 0, q_idx = *q_idx_p;
1441         struct Qdisc *q;
1442
1443         if (!root)
1444                 return 0;
1445
1446         q = root;
1447         if (q_idx < s_q_idx) {
1448                 q_idx++;
1449         } else {
1450                 if (!tc_qdisc_dump_ignore(q) &&
1451                     tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1452                                   cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1453                         goto done;
1454                 q_idx++;
1455         }
1456         list_for_each_entry(q, &root->list, list) {
1457                 if (q_idx < s_q_idx) {
1458                         q_idx++;
1459                         continue;
1460                 }
1461                 if (!tc_qdisc_dump_ignore(q) &&
1462                     tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1463                                   cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1464                         goto done;
1465                 q_idx++;
1466         }
1467
1468 out:
1469         *q_idx_p = q_idx;
1470         return ret;
1471 done:
1472         ret = -1;
1473         goto out;
1474 }
1475
1476 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1477 {
1478         struct net *net = sock_net(skb->sk);
1479         int idx, q_idx;
1480         int s_idx, s_q_idx;
1481         struct net_device *dev;
1482
1483         s_idx = cb->args[0];
1484         s_q_idx = q_idx = cb->args[1];
1485
1486         idx = 0;
1487         ASSERT_RTNL();
1488         for_each_netdev(net, dev) {
1489                 struct netdev_queue *dev_queue;
1490
1491                 if (idx < s_idx)
1492                         goto cont;
1493                 if (idx > s_idx)
1494                         s_q_idx = 0;
1495                 q_idx = 0;
1496
1497                 if (tc_dump_qdisc_root(dev->qdisc, skb, cb, &q_idx, s_q_idx) < 0)
1498                         goto done;
1499
1500                 dev_queue = dev_ingress_queue(dev);
1501                 if (dev_queue &&
1502                     tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
1503                                        &q_idx, s_q_idx) < 0)
1504                         goto done;
1505
1506 cont:
1507                 idx++;
1508         }
1509
1510 done:
1511         cb->args[0] = idx;
1512         cb->args[1] = q_idx;
1513
1514         return skb->len;
1515 }
1516
1517
1518
1519 /************************************************
1520  *      Traffic classes manipulation.           *
1521  ************************************************/
1522
1523
1524
1525 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n)
1526 {
1527         struct net *net = sock_net(skb->sk);
1528         struct tcmsg *tcm = nlmsg_data(n);
1529         struct nlattr *tca[TCA_MAX + 1];
1530         struct net_device *dev;
1531         struct Qdisc *q = NULL;
1532         const struct Qdisc_class_ops *cops;
1533         unsigned long cl = 0;
1534         unsigned long new_cl;
1535         u32 portid;
1536         u32 clid;
1537         u32 qid;
1538         int err;
1539
1540         if ((n->nlmsg_type != RTM_GETTCLASS) &&
1541             !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1542                 return -EPERM;
1543
1544         err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1545         if (err < 0)
1546                 return err;
1547
1548         dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1549         if (!dev)
1550                 return -ENODEV;
1551
1552         /*
1553            parent == TC_H_UNSPEC - unspecified parent.
1554            parent == TC_H_ROOT   - class is root, which has no parent.
1555            parent == X:0         - parent is root class.
1556            parent == X:Y         - parent is a node in hierarchy.
1557            parent == 0:Y         - parent is X:Y, where X:0 is qdisc.
1558
1559            handle == 0:0         - generate handle from kernel pool.
1560            handle == 0:Y         - class is X:Y, where X:0 is qdisc.
1561            handle == X:Y         - clear.
1562            handle == X:0         - root class.
1563          */
1564
1565         /* Step 1. Determine qdisc handle X:0 */
1566
1567         portid = tcm->tcm_parent;
1568         clid = tcm->tcm_handle;
1569         qid = TC_H_MAJ(clid);
1570
1571         if (portid != TC_H_ROOT) {
1572                 u32 qid1 = TC_H_MAJ(portid);
1573
1574                 if (qid && qid1) {
1575                         /* If both majors are known, they must be identical. */
1576                         if (qid != qid1)
1577                                 return -EINVAL;
1578                 } else if (qid1) {
1579                         qid = qid1;
1580                 } else if (qid == 0)
1581                         qid = dev->qdisc->handle;
1582
1583                 /* Now qid is genuine qdisc handle consistent
1584                  * both with parent and child.
1585                  *
1586                  * TC_H_MAJ(portid) still may be unspecified, complete it now.
1587                  */
1588                 if (portid)
1589                         portid = TC_H_MAKE(qid, portid);
1590         } else {
1591                 if (qid == 0)
1592                         qid = dev->qdisc->handle;
1593         }
1594
1595         /* OK. Locate qdisc */
1596         q = qdisc_lookup(dev, qid);
1597         if (!q)
1598                 return -ENOENT;
1599
1600         /* An check that it supports classes */
1601         cops = q->ops->cl_ops;
1602         if (cops == NULL)
1603                 return -EINVAL;
1604
1605         /* Now try to get class */
1606         if (clid == 0) {
1607                 if (portid == TC_H_ROOT)
1608                         clid = qid;
1609         } else
1610                 clid = TC_H_MAKE(qid, clid);
1611
1612         if (clid)
1613                 cl = cops->get(q, clid);
1614
1615         if (cl == 0) {
1616                 err = -ENOENT;
1617                 if (n->nlmsg_type != RTM_NEWTCLASS ||
1618                     !(n->nlmsg_flags & NLM_F_CREATE))
1619                         goto out;
1620         } else {
1621                 switch (n->nlmsg_type) {
1622                 case RTM_NEWTCLASS:
1623                         err = -EEXIST;
1624                         if (n->nlmsg_flags & NLM_F_EXCL)
1625                                 goto out;
1626                         break;
1627                 case RTM_DELTCLASS:
1628                         err = -EOPNOTSUPP;
1629                         if (cops->delete)
1630                                 err = cops->delete(q, cl);
1631                         if (err == 0)
1632                                 tclass_notify(net, skb, n, q, cl, RTM_DELTCLASS);
1633                         goto out;
1634                 case RTM_GETTCLASS:
1635                         err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
1636                         goto out;
1637                 default:
1638                         err = -EINVAL;
1639                         goto out;
1640                 }
1641         }
1642
1643         new_cl = cl;
1644         err = -EOPNOTSUPP;
1645         if (cops->change)
1646                 err = cops->change(q, clid, portid, tca, &new_cl);
1647         if (err == 0)
1648                 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
1649
1650 out:
1651         if (cl)
1652                 cops->put(q, cl);
1653
1654         return err;
1655 }
1656
1657
1658 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1659                           unsigned long cl,
1660                           u32 portid, u32 seq, u16 flags, int event)
1661 {
1662         struct tcmsg *tcm;
1663         struct nlmsghdr  *nlh;
1664         unsigned char *b = skb_tail_pointer(skb);
1665         struct gnet_dump d;
1666         const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1667
1668         cond_resched();
1669         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1670         if (!nlh)
1671                 goto out_nlmsg_trim;
1672         tcm = nlmsg_data(nlh);
1673         tcm->tcm_family = AF_UNSPEC;
1674         tcm->tcm__pad1 = 0;
1675         tcm->tcm__pad2 = 0;
1676         tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1677         tcm->tcm_parent = q->handle;
1678         tcm->tcm_handle = q->handle;
1679         tcm->tcm_info = 0;
1680         if (nla_put_string(skb, TCA_KIND, q->ops->id))
1681                 goto nla_put_failure;
1682         if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1683                 goto nla_put_failure;
1684
1685         if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1686                                          qdisc_root_sleeping_lock(q), &d) < 0)
1687                 goto nla_put_failure;
1688
1689         if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1690                 goto nla_put_failure;
1691
1692         if (gnet_stats_finish_copy(&d) < 0)
1693                 goto nla_put_failure;
1694
1695         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1696         return skb->len;
1697
1698 out_nlmsg_trim:
1699 nla_put_failure:
1700         nlmsg_trim(skb, b);
1701         return -1;
1702 }
1703
1704 static int tclass_notify(struct net *net, struct sk_buff *oskb,
1705                          struct nlmsghdr *n, struct Qdisc *q,
1706                          unsigned long cl, int event)
1707 {
1708         struct sk_buff *skb;
1709         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1710
1711         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1712         if (!skb)
1713                 return -ENOBUFS;
1714
1715         if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event) < 0) {
1716                 kfree_skb(skb);
1717                 return -EINVAL;
1718         }
1719
1720         return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1721                               n->nlmsg_flags & NLM_F_ECHO);
1722 }
1723
1724 struct qdisc_dump_args {
1725         struct qdisc_walker     w;
1726         struct sk_buff          *skb;
1727         struct netlink_callback *cb;
1728 };
1729
1730 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walker *arg)
1731 {
1732         struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
1733
1734         return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid,
1735                               a->cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWTCLASS);
1736 }
1737
1738 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
1739                                 struct tcmsg *tcm, struct netlink_callback *cb,
1740                                 int *t_p, int s_t)
1741 {
1742         struct qdisc_dump_args arg;
1743
1744         if (tc_qdisc_dump_ignore(q) ||
1745             *t_p < s_t || !q->ops->cl_ops ||
1746             (tcm->tcm_parent &&
1747              TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
1748                 (*t_p)++;
1749                 return 0;
1750         }
1751         if (*t_p > s_t)
1752                 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
1753         arg.w.fn = qdisc_class_dump;
1754         arg.skb = skb;
1755         arg.cb = cb;
1756         arg.w.stop  = 0;
1757         arg.w.skip = cb->args[1];
1758         arg.w.count = 0;
1759         q->ops->cl_ops->walk(q, &arg.w);
1760         cb->args[1] = arg.w.count;
1761         if (arg.w.stop)
1762                 return -1;
1763         (*t_p)++;
1764         return 0;
1765 }
1766
1767 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
1768                                struct tcmsg *tcm, struct netlink_callback *cb,
1769                                int *t_p, int s_t)
1770 {
1771         struct Qdisc *q;
1772
1773         if (!root)
1774                 return 0;
1775
1776         if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
1777                 return -1;
1778
1779         list_for_each_entry(q, &root->list, list) {
1780                 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
1781                         return -1;
1782         }
1783
1784         return 0;
1785 }
1786
1787 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
1788 {
1789         struct tcmsg *tcm = nlmsg_data(cb->nlh);
1790         struct net *net = sock_net(skb->sk);
1791         struct netdev_queue *dev_queue;
1792         struct net_device *dev;
1793         int t, s_t;
1794
1795         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
1796                 return 0;
1797         dev = dev_get_by_index(net, tcm->tcm_ifindex);
1798         if (!dev)
1799                 return 0;
1800
1801         s_t = cb->args[0];
1802         t = 0;
1803
1804         if (tc_dump_tclass_root(dev->qdisc, skb, tcm, cb, &t, s_t) < 0)
1805                 goto done;
1806
1807         dev_queue = dev_ingress_queue(dev);
1808         if (dev_queue &&
1809             tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb,
1810                                 &t, s_t) < 0)
1811                 goto done;
1812
1813 done:
1814         cb->args[0] = t;
1815
1816         dev_put(dev);
1817         return skb->len;
1818 }
1819
1820 /* Main classifier routine: scans classifier chain attached
1821  * to this qdisc, (optionally) tests for protocol and asks
1822  * specific classifiers.
1823  */
1824 int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1825                 struct tcf_result *res, bool compat_mode)
1826 {
1827 #ifdef CONFIG_NET_CLS_ACT
1828         const struct tcf_proto *old_tp = tp;
1829         int limit = 0;
1830
1831 reclassify:
1832 #endif
1833         for (; tp; tp = rcu_dereference_bh(tp->next)) {
1834                 __be16 protocol = tc_skb_protocol(skb);
1835                 int err;
1836
1837                 if (tp->protocol != protocol &&
1838                     tp->protocol != htons(ETH_P_ALL))
1839                         continue;
1840
1841                 err = tp->classify(skb, tp, res);
1842 #ifdef CONFIG_NET_CLS_ACT
1843                 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode))
1844                         goto reset;
1845 #endif
1846                 if (err >= 0)
1847                         return err;
1848         }
1849
1850         return -1;
1851 #ifdef CONFIG_NET_CLS_ACT
1852 reset:
1853         if (unlikely(limit++ >= MAX_REC_LOOP)) {
1854                 net_notice_ratelimited("%s: reclassify loop, rule prio %u, protocol %02x\n",
1855                                        tp->q->ops->id, tp->prio & 0xffff,
1856                                        ntohs(tp->protocol));
1857                 return TC_ACT_SHOT;
1858         }
1859
1860         tp = old_tp;
1861         goto reclassify;
1862 #endif
1863 }
1864 EXPORT_SYMBOL(tc_classify);
1865
1866 bool tcf_destroy(struct tcf_proto *tp, bool force)
1867 {
1868         if (tp->ops->destroy(tp, force)) {
1869                 module_put(tp->ops->owner);
1870                 kfree_rcu(tp, rcu);
1871                 return true;
1872         }
1873
1874         return false;
1875 }
1876
1877 void tcf_destroy_chain(struct tcf_proto __rcu **fl)
1878 {
1879         struct tcf_proto *tp;
1880
1881         while ((tp = rtnl_dereference(*fl)) != NULL) {
1882                 RCU_INIT_POINTER(*fl, tp->next);
1883                 tcf_destroy(tp, true);
1884         }
1885 }
1886 EXPORT_SYMBOL(tcf_destroy_chain);
1887
1888 #ifdef CONFIG_PROC_FS
1889 static int psched_show(struct seq_file *seq, void *v)
1890 {
1891         seq_printf(seq, "%08x %08x %08x %08x\n",
1892                    (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
1893                    1000000,
1894                    (u32)NSEC_PER_SEC / hrtimer_resolution);
1895
1896         return 0;
1897 }
1898
1899 static int psched_open(struct inode *inode, struct file *file)
1900 {
1901         return single_open(file, psched_show, NULL);
1902 }
1903
1904 static const struct file_operations psched_fops = {
1905         .owner = THIS_MODULE,
1906         .open = psched_open,
1907         .read  = seq_read,
1908         .llseek = seq_lseek,
1909         .release = single_release,
1910 };
1911
1912 static int __net_init psched_net_init(struct net *net)
1913 {
1914         struct proc_dir_entry *e;
1915
1916         e = proc_create("psched", 0, net->proc_net, &psched_fops);
1917         if (e == NULL)
1918                 return -ENOMEM;
1919
1920         return 0;
1921 }
1922
1923 static void __net_exit psched_net_exit(struct net *net)
1924 {
1925         remove_proc_entry("psched", net->proc_net);
1926 }
1927 #else
1928 static int __net_init psched_net_init(struct net *net)
1929 {
1930         return 0;
1931 }
1932
1933 static void __net_exit psched_net_exit(struct net *net)
1934 {
1935 }
1936 #endif
1937
1938 static struct pernet_operations psched_net_ops = {
1939         .init = psched_net_init,
1940         .exit = psched_net_exit,
1941 };
1942
1943 static int __init pktsched_init(void)
1944 {
1945         int err;
1946
1947         err = register_pernet_subsys(&psched_net_ops);
1948         if (err) {
1949                 pr_err("pktsched_init: "
1950                        "cannot initialize per netns operations\n");
1951                 return err;
1952         }
1953
1954         register_qdisc(&pfifo_fast_ops);
1955         register_qdisc(&pfifo_qdisc_ops);
1956         register_qdisc(&bfifo_qdisc_ops);
1957         register_qdisc(&pfifo_head_drop_qdisc_ops);
1958         register_qdisc(&mq_qdisc_ops);
1959         register_qdisc(&noqueue_qdisc_ops);
1960
1961         rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, NULL);
1962         rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, NULL);
1963         rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc, NULL);
1964         rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, NULL);
1965         rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, NULL);
1966         rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass, NULL);
1967
1968         return 0;
1969 }
1970
1971 subsys_initcall(pktsched_init);