net/sched/act_skbedit.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2008, Intel Corporation.
   4  *
   5  * Author: Alexander Duyck <alexander.h.duyck@intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/kernel.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/rtnetlink.h>
  13 #include <net/netlink.h>
  14 #include <net/pkt_sched.h>
  15 #include <net/ip.h>
  16 #include <net/ipv6.h>
  17 #include <net/dsfield.h>
  18 #include <net/pkt_cls.h>
  19
  20 #include <linux/tc_act/tc_skbedit.h>
  21 #include <net/tc_act/tc_skbedit.h>
  22
  23 static unsigned int skbedit_net_id;
  24 static struct tc_action_ops act_skbedit_ops;
  25
  26 static u16 tcf_skbedit_hash(struct tcf_skbedit_params *params,
  27                             struct sk_buff *skb)
  28 {
  29         u16 queue_mapping = params->queue_mapping;
  30
  31         if (params->flags & SKBEDIT_F_TXQ_SKBHASH) {
  32                 u32 hash = skb_get_hash(skb);
  33
  34                 queue_mapping += hash % params->mapping_mod;
  35         }
  36
  37         return netdev_cap_txqueue(skb->dev, queue_mapping);
  38 }
  39
  40 static int tcf_skbedit_act(struct sk_buff *skb, const struct tc_action *a,
  41                            struct tcf_result *res)
  42 {
  43         struct tcf_skbedit *d = to_skbedit(a);
  44         struct tcf_skbedit_params *params;
  45         int action;
  46
  47         tcf_lastuse_update(&d->tcf_tm);
  48         bstats_update(this_cpu_ptr(d->common.cpu_bstats), skb);
  49
  50         params = rcu_dereference_bh(d->params);
  51         action = READ_ONCE(d->tcf_action);
  52
  53         if (params->flags & SKBEDIT_F_PRIORITY)
  54                 skb->priority = params->priority;
  55         if (params->flags & SKBEDIT_F_INHERITDSFIELD) {
  56                 int wlen = skb_network_offset(skb);
  57
  58                 switch (skb_protocol(skb, true)) {
  59                 case htons(ETH_P_IP):
  60                         wlen += sizeof(struct iphdr);
  61                         if (!pskb_may_pull(skb, wlen))
  62                                 goto err;
  63                         skb->priority = ipv4_get_dsfield(ip_hdr(skb)) >> 2;
  64                         break;
  65
  66                 case htons(ETH_P_IPV6):
  67                         wlen += sizeof(struct ipv6hdr);
  68                         if (!pskb_may_pull(skb, wlen))
  69                                 goto err;
  70                         skb->priority = ipv6_get_dsfield(ipv6_hdr(skb)) >> 2;
  71                         break;
  72                 }
  73         }
  74         if (params->flags & SKBEDIT_F_QUEUE_MAPPING &&
  75             skb->dev->real_num_tx_queues > params->queue_mapping) {
  76 #ifdef CONFIG_NET_EGRESS
  77                 netdev_xmit_skip_txqueue(true);
  78 #endif
  79                 skb_set_queue_mapping(skb, tcf_skbedit_hash(params, skb));
  80         }
  81         if (params->flags & SKBEDIT_F_MARK) {
  82                 skb->mark &= ~params->mask;
  83                 skb->mark |= params->mark & params->mask;
  84         }
  85         if (params->flags & SKBEDIT_F_PTYPE)
  86                 skb->pkt_type = params->ptype;
  87         return action;
  88
  89 err:
  90         qstats_drop_inc(this_cpu_ptr(d->common.cpu_qstats));
  91         return TC_ACT_SHOT;
  92 }
  93
  94 static void tcf_skbedit_stats_update(struct tc_action *a, u64 bytes,
  95                                      u64 packets, u64 drops,
  96                                      u64 lastuse, bool hw)
  97 {
  98         struct tcf_skbedit *d = to_skbedit(a);
  99         struct tcf_t *tm = &d->tcf_tm;
 100
 101         tcf_action_update_stats(a, bytes, packets, drops, hw);
 102         tm->lastuse = max_t(u64, tm->lastuse, lastuse);
 103 }
 104
 105 static const struct nla_policy skbedit_policy[TCA_SKBEDIT_MAX + 1] = {
 106         [TCA_SKBEDIT_PARMS]             = { .len = sizeof(struct tc_skbedit) },
 107         [TCA_SKBEDIT_PRIORITY]          = { .len = sizeof(u32) },
 108         [TCA_SKBEDIT_QUEUE_MAPPING]     = { .len = sizeof(u16) },
 109         [TCA_SKBEDIT_MARK]              = { .len = sizeof(u32) },
 110         [TCA_SKBEDIT_PTYPE]             = { .len = sizeof(u16) },
 111         [TCA_SKBEDIT_MASK]              = { .len = sizeof(u32) },
 112         [TCA_SKBEDIT_FLAGS]             = { .len = sizeof(u64) },
 113         [TCA_SKBEDIT_QUEUE_MAPPING_MAX] = { .len = sizeof(u16) },
 114 };
 115
 116 static int tcf_skbedit_init(struct net *net, struct nlattr *nla,
 117                             struct nlattr *est, struct tc_action **a,
 118                             struct tcf_proto *tp, u32 act_flags,
 119                             struct netlink_ext_ack *extack)
 120 {
 121         struct tc_action_net *tn = net_generic(net, skbedit_net_id);
 122         bool bind = act_flags & TCA_ACT_FLAGS_BIND;
 123         struct tcf_skbedit_params *params_new;
 124         struct nlattr *tb[TCA_SKBEDIT_MAX + 1];
 125         struct tcf_chain *goto_ch = NULL;
 126         struct tc_skbedit *parm;
 127         struct tcf_skbedit *d;
 128         u32 flags = 0, *priority = NULL, *mark = NULL, *mask = NULL;
 129         u16 *queue_mapping = NULL, *ptype = NULL;
 130         u16 mapping_mod = 1;
 131         bool exists = false;
 132         int ret = 0, err;
 133         u32 index;
 134
 135         if (nla == NULL)
 136                 return -EINVAL;
 137
 138         err = nla_parse_nested_deprecated(tb, TCA_SKBEDIT_MAX, nla,
 139                                           skbedit_policy, NULL);
 140         if (err < 0)
 141                 return err;
 142
 143         if (tb[TCA_SKBEDIT_PARMS] == NULL)
 144                 return -EINVAL;
 145
 146         if (tb[TCA_SKBEDIT_PRIORITY] != NULL) {
 147                 flags |= SKBEDIT_F_PRIORITY;
 148                 priority = nla_data(tb[TCA_SKBEDIT_PRIORITY]);
 149         }
 150
 151         if (tb[TCA_SKBEDIT_QUEUE_MAPPING] != NULL) {
 152                 flags |= SKBEDIT_F_QUEUE_MAPPING;
 153                 queue_mapping = nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING]);
 154         }
 155
 156         if (tb[TCA_SKBEDIT_PTYPE] != NULL) {
 157                 ptype = nla_data(tb[TCA_SKBEDIT_PTYPE]);
 158                 if (!skb_pkt_type_ok(*ptype))
 159                         return -EINVAL;
 160                 flags |= SKBEDIT_F_PTYPE;
 161         }
 162
 163         if (tb[TCA_SKBEDIT_MARK] != NULL) {
 164                 flags |= SKBEDIT_F_MARK;
 165                 mark = nla_data(tb[TCA_SKBEDIT_MARK]);
 166         }
 167
 168         if (tb[TCA_SKBEDIT_MASK] != NULL) {
 169                 flags |= SKBEDIT_F_MASK;
 170                 mask = nla_data(tb[TCA_SKBEDIT_MASK]);
 171         }
 172
 173         if (tb[TCA_SKBEDIT_FLAGS] != NULL) {
 174                 u64 *pure_flags = nla_data(tb[TCA_SKBEDIT_FLAGS]);
 175
 176                 if (*pure_flags & SKBEDIT_F_TXQ_SKBHASH) {
 177                         u16 *queue_mapping_max;
 178
 179                         if (!tb[TCA_SKBEDIT_QUEUE_MAPPING] ||
 180                             !tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]) {
 181                                 NL_SET_ERR_MSG_MOD(extack, "Missing required range of queue_mapping.");
 182                                 return -EINVAL;
 183                         }
 184
 185                         queue_mapping_max =
 186                                 nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]);
 187                         if (*queue_mapping_max < *queue_mapping) {
 188                                 NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid, max < min.");
 189                                 return -EINVAL;
 190                         }
 191
 192                         mapping_mod = *queue_mapping_max - *queue_mapping + 1;
 193                         flags |= SKBEDIT_F_TXQ_SKBHASH;
 194                 }
 195                 if (*pure_flags & SKBEDIT_F_INHERITDSFIELD)
 196                         flags |= SKBEDIT_F_INHERITDSFIELD;
 197         }
 198
 199         parm = nla_data(tb[TCA_SKBEDIT_PARMS]);
 200         index = parm->index;
 201         err = tcf_idr_check_alloc(tn, &index, a, bind);
 202         if (err < 0)
 203                 return err;
 204         exists = err;
 205         if (exists && bind)
 206                 return 0;
 207
 208         if (!flags) {
 209                 if (exists)
 210                         tcf_idr_release(*a, bind);
 211                 else
 212                         tcf_idr_cleanup(tn, index);
 213                 return -EINVAL;
 214         }
 215
 216         if (!exists) {
 217                 ret = tcf_idr_create(tn, index, est, a,
 218                                      &act_skbedit_ops, bind, true, act_flags);
 219                 if (ret) {
 220                         tcf_idr_cleanup(tn, index);
 221                         return ret;
 222                 }
 223
 224                 d = to_skbedit(*a);
 225                 ret = ACT_P_CREATED;
 226         } else {
 227                 d = to_skbedit(*a);
 228                 if (!(act_flags & TCA_ACT_FLAGS_REPLACE)) {
 229                         tcf_idr_release(*a, bind);
 230                         return -EEXIST;
 231                 }
 232         }
 233         err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
 234         if (err < 0)
 235                 goto release_idr;
 236
 237         params_new = kzalloc(sizeof(*params_new), GFP_KERNEL);
 238         if (unlikely(!params_new)) {
 239                 err = -ENOMEM;
 240                 goto put_chain;
 241         }
 242
 243         params_new->flags = flags;
 244         if (flags & SKBEDIT_F_PRIORITY)
 245                 params_new->priority = *priority;
 246         if (flags & SKBEDIT_F_QUEUE_MAPPING) {
 247                 params_new->queue_mapping = *queue_mapping;
 248                 params_new->mapping_mod = mapping_mod;
 249         }
 250         if (flags & SKBEDIT_F_MARK)
 251                 params_new->mark = *mark;
 252         if (flags & SKBEDIT_F_PTYPE)
 253                 params_new->ptype = *ptype;
 254         /* default behaviour is to use all the bits */
 255         params_new->mask = 0xffffffff;
 256         if (flags & SKBEDIT_F_MASK)
 257                 params_new->mask = *mask;
 258
 259         spin_lock_bh(&d->tcf_lock);
 260         goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
 261         params_new = rcu_replace_pointer(d->params, params_new,
 262                                          lockdep_is_held(&d->tcf_lock));
 263         spin_unlock_bh(&d->tcf_lock);
 264         if (params_new)
 265                 kfree_rcu(params_new, rcu);
 266         if (goto_ch)
 267                 tcf_chain_put_by_act(goto_ch);
 268
 269         return ret;
 270 put_chain:
 271         if (goto_ch)
 272                 tcf_chain_put_by_act(goto_ch);
 273 release_idr:
 274         tcf_idr_release(*a, bind);
 275         return err;
 276 }
 277
 278 static int tcf_skbedit_dump(struct sk_buff *skb, struct tc_action *a,
 279                             int bind, int ref)
 280 {
 281         unsigned char *b = skb_tail_pointer(skb);
 282         struct tcf_skbedit *d = to_skbedit(a);
 283         struct tcf_skbedit_params *params;
 284         struct tc_skbedit opt = {
 285                 .index   = d->tcf_index,
 286                 .refcnt  = refcount_read(&d->tcf_refcnt) - ref,
 287                 .bindcnt = atomic_read(&d->tcf_bindcnt) - bind,
 288         };
 289         u64 pure_flags = 0;
 290         struct tcf_t t;
 291
 292         spin_lock_bh(&d->tcf_lock);
 293         params = rcu_dereference_protected(d->params,
 294                                            lockdep_is_held(&d->tcf_lock));
 295         opt.action = d->tcf_action;
 296
 297         if (nla_put(skb, TCA_SKBEDIT_PARMS, sizeof(opt), &opt))
 298                 goto nla_put_failure;
 299         if ((params->flags & SKBEDIT_F_PRIORITY) &&
 300             nla_put_u32(skb, TCA_SKBEDIT_PRIORITY, params->priority))
 301                 goto nla_put_failure;
 302         if ((params->flags & SKBEDIT_F_QUEUE_MAPPING) &&
 303             nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING, params->queue_mapping))
 304                 goto nla_put_failure;
 305         if ((params->flags & SKBEDIT_F_MARK) &&
 306             nla_put_u32(skb, TCA_SKBEDIT_MARK, params->mark))
 307                 goto nla_put_failure;
 308         if ((params->flags & SKBEDIT_F_PTYPE) &&
 309             nla_put_u16(skb, TCA_SKBEDIT_PTYPE, params->ptype))
 310                 goto nla_put_failure;
 311         if ((params->flags & SKBEDIT_F_MASK) &&
 312             nla_put_u32(skb, TCA_SKBEDIT_MASK, params->mask))
 313                 goto nla_put_failure;
 314         if (params->flags & SKBEDIT_F_INHERITDSFIELD)
 315                 pure_flags |= SKBEDIT_F_INHERITDSFIELD;
 316         if (params->flags & SKBEDIT_F_TXQ_SKBHASH) {
 317                 if (nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING_MAX,
 318                                 params->queue_mapping + params->mapping_mod - 1))
 319                         goto nla_put_failure;
 320
 321                 pure_flags |= SKBEDIT_F_TXQ_SKBHASH;
 322         }
 323         if (pure_flags != 0 &&
 324             nla_put(skb, TCA_SKBEDIT_FLAGS, sizeof(pure_flags), &pure_flags))
 325                 goto nla_put_failure;
 326
 327         tcf_tm_dump(&t, &d->tcf_tm);
 328         if (nla_put_64bit(skb, TCA_SKBEDIT_TM, sizeof(t), &t, TCA_SKBEDIT_PAD))
 329                 goto nla_put_failure;
 330         spin_unlock_bh(&d->tcf_lock);
 331
 332         return skb->len;
 333
 334 nla_put_failure:
 335         spin_unlock_bh(&d->tcf_lock);
 336         nlmsg_trim(skb, b);
 337         return -1;
 338 }
 339
 340 static void tcf_skbedit_cleanup(struct tc_action *a)
 341 {
 342         struct tcf_skbedit *d = to_skbedit(a);
 343         struct tcf_skbedit_params *params;
 344
 345         params = rcu_dereference_protected(d->params, 1);
 346         if (params)
 347                 kfree_rcu(params, rcu);
 348 }
 349
 350 static int tcf_skbedit_walker(struct net *net, struct sk_buff *skb,
 351                               struct netlink_callback *cb, int type,
 352                               const struct tc_action_ops *ops,
 353                               struct netlink_ext_ack *extack)
 354 {
 355         struct tc_action_net *tn = net_generic(net, skbedit_net_id);
 356
 357         return tcf_generic_walker(tn, skb, cb, type, ops, extack);
 358 }
 359
 360 static int tcf_skbedit_search(struct net *net, struct tc_action **a, u32 index)
 361 {
 362         struct tc_action_net *tn = net_generic(net, skbedit_net_id);
 363
 364         return tcf_idr_search(tn, a, index);
 365 }
 366
 367 static size_t tcf_skbedit_get_fill_size(const struct tc_action *act)
 368 {
 369         return nla_total_size(sizeof(struct tc_skbedit))
 370                 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_PRIORITY */
 371                 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING */
 372                 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING_MAX */
 373                 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MARK */
 374                 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_PTYPE */
 375                 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MASK */
 376                 + nla_total_size_64bit(sizeof(u64)); /* TCA_SKBEDIT_FLAGS */
 377 }
 378
 379 static int tcf_skbedit_offload_act_setup(struct tc_action *act, void *entry_data,
 380                                          u32 *index_inc, bool bind,
 381                                          struct netlink_ext_ack *extack)
 382 {
 383         if (bind) {
 384                 struct flow_action_entry *entry = entry_data;
 385
 386                 if (is_tcf_skbedit_mark(act)) {
 387                         entry->id = FLOW_ACTION_MARK;
 388                         entry->mark = tcf_skbedit_mark(act);
 389                 } else if (is_tcf_skbedit_ptype(act)) {
 390                         entry->id = FLOW_ACTION_PTYPE;
 391                         entry->ptype = tcf_skbedit_ptype(act);
 392                 } else if (is_tcf_skbedit_priority(act)) {
 393                         entry->id = FLOW_ACTION_PRIORITY;
 394                         entry->priority = tcf_skbedit_priority(act);
 395                 } else if (is_tcf_skbedit_queue_mapping(act)) {
 396                         NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"queue_mapping\" option is used");
 397                         return -EOPNOTSUPP;
 398                 } else if (is_tcf_skbedit_inheritdsfield(act)) {
 399                         NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"inheritdsfield\" option is used");
 400                         return -EOPNOTSUPP;
 401                 } else {
 402                         NL_SET_ERR_MSG_MOD(extack, "Unsupported skbedit option offload");
 403                         return -EOPNOTSUPP;
 404                 }
 405                 *index_inc = 1;
 406         } else {
 407                 struct flow_offload_action *fl_action = entry_data;
 408
 409                 if (is_tcf_skbedit_mark(act))
 410                         fl_action->id = FLOW_ACTION_MARK;
 411                 else if (is_tcf_skbedit_ptype(act))
 412                         fl_action->id = FLOW_ACTION_PTYPE;
 413                 else if (is_tcf_skbedit_priority(act))
 414                         fl_action->id = FLOW_ACTION_PRIORITY;
 415                 else
 416                         return -EOPNOTSUPP;
 417         }
 418
 419         return 0;
 420 }
 421
 422 static struct tc_action_ops act_skbedit_ops = {
 423         .kind           =       "skbedit",
 424         .id             =       TCA_ID_SKBEDIT,
 425         .owner          =       THIS_MODULE,
 426         .act            =       tcf_skbedit_act,
 427         .stats_update   =       tcf_skbedit_stats_update,
 428         .dump           =       tcf_skbedit_dump,
 429         .init           =       tcf_skbedit_init,
 430         .cleanup        =       tcf_skbedit_cleanup,
 431         .walk           =       tcf_skbedit_walker,
 432         .get_fill_size  =       tcf_skbedit_get_fill_size,
 433         .lookup         =       tcf_skbedit_search,
 434         .offload_act_setup =    tcf_skbedit_offload_act_setup,
 435         .size           =       sizeof(struct tcf_skbedit),
 436 };
 437
 438 static __net_init int skbedit_init_net(struct net *net)
 439 {
 440         struct tc_action_net *tn = net_generic(net, skbedit_net_id);
 441
 442         return tc_action_net_init(net, tn, &act_skbedit_ops);
 443 }
 444
 445 static void __net_exit skbedit_exit_net(struct list_head *net_list)
 446 {
 447         tc_action_net_exit(net_list, skbedit_net_id);
 448 }
 449
 450 static struct pernet_operations skbedit_net_ops = {
 451         .init = skbedit_init_net,
 452         .exit_batch = skbedit_exit_net,
 453         .id   = &skbedit_net_id,
 454         .size = sizeof(struct tc_action_net),
 455 };
 456
 457 MODULE_AUTHOR("Alexander Duyck, <alexander.h.duyck@intel.com>");
 458 MODULE_DESCRIPTION("SKB Editing");
 459 MODULE_LICENSE("GPL");
 460
 461 static int __init skbedit_init_module(void)
 462 {
 463         return tcf_register_action(&act_skbedit_ops, &skbedit_net_ops);
 464 }
 465
 466 static void __exit skbedit_cleanup_module(void)
 467 {
 468         tcf_unregister_action(&act_skbedit_ops, &skbedit_net_ops);
 469 }
 470
 471 module_init(skbedit_init_module);
 472 module_exit(skbedit_cleanup_module);