1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <linux/init.h>
3 #include <linux/module.h>
4 #include <linux/netfilter.h>
5 #include <net/flow_offload.h>
6 #include <net/netfilter/nf_tables.h>
7 #include <net/netfilter/nf_tables_offload.h>
8 #include <net/pkt_cls.h>
10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
12 struct nft_flow_rule *flow;
14 flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
18 flow->rule = flow_rule_alloc(num_actions);
24 flow->rule->match.dissector = &flow->match.dissector;
25 flow->rule->match.mask = &flow->match.mask;
26 flow->rule->match.key = &flow->match.key;
31 void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
32 enum flow_dissector_key_id addr_type)
34 struct nft_flow_match *match = &flow->match;
35 struct nft_flow_key *mask = &match->mask;
36 struct nft_flow_key *key = &match->key;
38 if (match->dissector.used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL))
41 key->control.addr_type = addr_type;
42 mask->control.addr_type = 0xffff;
43 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL);
44 match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] =
45 offsetof(struct nft_flow_key, control);
48 struct nft_offload_ethertype {
53 static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx,
54 struct nft_flow_rule *flow)
56 struct nft_flow_match *match = &flow->match;
57 struct nft_offload_ethertype ethertype = {
58 .value = match->key.basic.n_proto,
59 .mask = match->mask.basic.n_proto,
62 if (match->dissector.used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) &&
63 (match->key.vlan.vlan_tpid == htons(ETH_P_8021Q) ||
64 match->key.vlan.vlan_tpid == htons(ETH_P_8021AD))) {
65 match->key.basic.n_proto = match->key.cvlan.vlan_tpid;
66 match->mask.basic.n_proto = match->mask.cvlan.vlan_tpid;
67 match->key.cvlan.vlan_tpid = match->key.vlan.vlan_tpid;
68 match->mask.cvlan.vlan_tpid = match->mask.vlan.vlan_tpid;
69 match->key.vlan.vlan_tpid = ethertype.value;
70 match->mask.vlan.vlan_tpid = ethertype.mask;
71 match->dissector.offset[FLOW_DISSECTOR_KEY_CVLAN] =
72 offsetof(struct nft_flow_key, cvlan);
73 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN);
74 } else if (match->dissector.used_keys &
75 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) &&
76 (match->key.basic.n_proto == htons(ETH_P_8021Q) ||
77 match->key.basic.n_proto == htons(ETH_P_8021AD))) {
78 match->key.basic.n_proto = match->key.vlan.vlan_tpid;
79 match->mask.basic.n_proto = match->mask.vlan.vlan_tpid;
80 match->key.vlan.vlan_tpid = ethertype.value;
81 match->mask.vlan.vlan_tpid = ethertype.mask;
82 match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] =
83 offsetof(struct nft_flow_key, vlan);
84 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_VLAN);
88 struct nft_flow_rule *nft_flow_rule_create(struct net *net,
89 const struct nft_rule *rule)
91 struct nft_offload_ctx *ctx;
92 struct nft_flow_rule *flow;
93 int num_actions = 0, err;
94 struct nft_expr *expr;
96 expr = nft_expr_first(rule);
97 while (nft_expr_more(rule, expr)) {
98 if (expr->ops->offload_action &&
99 expr->ops->offload_action(expr))
102 expr = nft_expr_next(expr);
105 if (num_actions == 0)
106 return ERR_PTR(-EOPNOTSUPP);
108 flow = nft_flow_rule_alloc(num_actions);
110 return ERR_PTR(-ENOMEM);
112 expr = nft_expr_first(rule);
114 ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
120 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
122 while (nft_expr_more(rule, expr)) {
123 if (!expr->ops->offload) {
127 err = expr->ops->offload(ctx, flow, expr);
131 expr = nft_expr_next(expr);
133 nft_flow_rule_transfer_vlan(ctx, flow);
135 flow->proto = ctx->dep.l3num;
141 nft_flow_rule_destroy(flow);
146 void nft_flow_rule_destroy(struct nft_flow_rule *flow)
148 struct flow_action_entry *entry;
151 flow_action_for_each(i, entry, &flow->rule->action) {
153 case FLOW_ACTION_REDIRECT:
154 case FLOW_ACTION_MIRRED:
165 void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
166 enum nft_offload_dep_type type)
168 ctx->dep.type = type;
171 void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
172 const void *data, u32 len)
174 switch (ctx->dep.type) {
175 case NFT_OFFLOAD_DEP_NETWORK:
176 WARN_ON(len != sizeof(__u16));
177 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
179 case NFT_OFFLOAD_DEP_TRANSPORT:
180 WARN_ON(len != sizeof(__u8));
181 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
186 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
189 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
190 __be16 proto, int priority,
191 struct netlink_ext_ack *extack)
193 common->protocol = proto;
194 common->prio = priority;
195 common->extack = extack;
198 static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
199 struct list_head *cb_list)
201 struct flow_block_cb *block_cb;
204 list_for_each_entry(block_cb, cb_list, list) {
205 err = block_cb->cb(type, type_data, block_cb->cb_priv);
212 static int nft_chain_offload_priority(const struct nft_base_chain *basechain)
214 if (basechain->ops.priority <= 0 ||
215 basechain->ops.priority > USHRT_MAX)
221 bool nft_chain_offload_support(const struct nft_base_chain *basechain)
223 struct net_device *dev;
224 struct nft_hook *hook;
226 if (nft_chain_offload_priority(basechain) < 0)
229 list_for_each_entry(hook, &basechain->hook_list, list) {
230 if (hook->ops.pf != NFPROTO_NETDEV ||
231 hook->ops.hooknum != NF_NETDEV_INGRESS)
235 if (!dev->netdev_ops->ndo_setup_tc && !flow_indr_dev_exists())
242 static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
243 const struct nft_base_chain *basechain,
244 const struct nft_rule *rule,
245 const struct nft_flow_rule *flow,
246 struct netlink_ext_ack *extack,
247 enum flow_cls_command command)
249 __be16 proto = ETH_P_ALL;
251 memset(cls_flow, 0, sizeof(*cls_flow));
256 nft_flow_offload_common_init(&cls_flow->common, proto,
257 basechain->ops.priority, extack);
258 cls_flow->command = command;
259 cls_flow->cookie = (unsigned long) rule;
261 cls_flow->rule = flow->rule;
264 static int nft_flow_offload_cmd(const struct nft_chain *chain,
265 const struct nft_rule *rule,
266 struct nft_flow_rule *flow,
267 enum flow_cls_command command,
268 struct flow_cls_offload *cls_flow)
270 struct netlink_ext_ack extack = {};
271 struct nft_base_chain *basechain;
273 if (!nft_is_base_chain(chain))
276 basechain = nft_base_chain(chain);
277 nft_flow_cls_offload_setup(cls_flow, basechain, rule, flow, &extack,
280 return nft_setup_cb_call(TC_SETUP_CLSFLOWER, cls_flow,
281 &basechain->flow_block.cb_list);
284 static int nft_flow_offload_rule(const struct nft_chain *chain,
285 struct nft_rule *rule,
286 struct nft_flow_rule *flow,
287 enum flow_cls_command command)
289 struct flow_cls_offload cls_flow;
291 return nft_flow_offload_cmd(chain, rule, flow, command, &cls_flow);
294 int nft_flow_rule_stats(const struct nft_chain *chain,
295 const struct nft_rule *rule)
297 struct flow_cls_offload cls_flow = {};
298 struct nft_expr *expr, *next;
301 err = nft_flow_offload_cmd(chain, rule, NULL, FLOW_CLS_STATS,
306 nft_rule_for_each_expr(expr, next, rule) {
307 if (expr->ops->offload_stats)
308 expr->ops->offload_stats(expr, &cls_flow.stats);
314 static int nft_flow_offload_bind(struct flow_block_offload *bo,
315 struct nft_base_chain *basechain)
317 list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
321 static int nft_flow_offload_unbind(struct flow_block_offload *bo,
322 struct nft_base_chain *basechain)
324 struct flow_block_cb *block_cb, *next;
325 struct flow_cls_offload cls_flow;
326 struct netlink_ext_ack extack;
327 struct nft_chain *chain;
328 struct nft_rule *rule;
330 chain = &basechain->chain;
331 list_for_each_entry(rule, &chain->rules, list) {
332 memset(&extack, 0, sizeof(extack));
333 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
334 &extack, FLOW_CLS_DESTROY);
335 nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
338 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
339 list_del(&block_cb->list);
340 flow_block_cb_free(block_cb);
346 static int nft_block_setup(struct nft_base_chain *basechain,
347 struct flow_block_offload *bo,
348 enum flow_block_command cmd)
353 case FLOW_BLOCK_BIND:
354 err = nft_flow_offload_bind(bo, basechain);
356 case FLOW_BLOCK_UNBIND:
357 err = nft_flow_offload_unbind(bo, basechain);
367 static void nft_flow_block_offload_init(struct flow_block_offload *bo,
369 enum flow_block_command cmd,
370 struct nft_base_chain *basechain,
371 struct netlink_ext_ack *extack)
373 memset(bo, 0, sizeof(*bo));
375 bo->block = &basechain->flow_block;
377 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
379 bo->cb_list_head = &basechain->flow_block.cb_list;
380 INIT_LIST_HEAD(&bo->cb_list);
383 static int nft_block_offload_cmd(struct nft_base_chain *chain,
384 struct net_device *dev,
385 enum flow_block_command cmd)
387 struct netlink_ext_ack extack = {};
388 struct flow_block_offload bo;
391 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
393 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
397 return nft_block_setup(chain, &bo, cmd);
400 static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
402 struct nft_base_chain *basechain = block_cb->indr.data;
403 struct net_device *dev = block_cb->indr.dev;
404 struct netlink_ext_ack extack = {};
405 struct nftables_pernet *nft_net;
406 struct net *net = dev_net(dev);
407 struct flow_block_offload bo;
409 nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
411 nft_net = nft_pernet(net);
412 mutex_lock(&nft_net->commit_mutex);
413 list_del(&block_cb->driver_list);
414 list_move(&block_cb->list, &bo.cb_list);
415 nft_flow_offload_unbind(&bo, basechain);
416 mutex_unlock(&nft_net->commit_mutex);
419 static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
420 struct net_device *dev,
421 enum flow_block_command cmd)
423 struct netlink_ext_ack extack = {};
424 struct flow_block_offload bo;
427 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);
429 err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo,
430 nft_indr_block_cleanup);
434 if (list_empty(&bo.cb_list))
437 return nft_block_setup(basechain, &bo, cmd);
440 static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
441 struct net_device *dev,
442 enum flow_block_command cmd)
446 if (dev->netdev_ops->ndo_setup_tc)
447 err = nft_block_offload_cmd(basechain, dev, cmd);
449 err = nft_indr_block_offload_cmd(basechain, dev, cmd);
454 static int nft_flow_block_chain(struct nft_base_chain *basechain,
455 const struct net_device *this_dev,
456 enum flow_block_command cmd)
458 struct net_device *dev;
459 struct nft_hook *hook;
462 list_for_each_entry(hook, &basechain->hook_list, list) {
464 if (this_dev && this_dev != dev)
467 err = nft_chain_offload_cmd(basechain, dev, cmd);
468 if (err < 0 && cmd == FLOW_BLOCK_BIND) {
480 list_for_each_entry(hook, &basechain->hook_list, list) {
485 nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
490 static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
491 enum flow_block_command cmd)
493 struct nft_base_chain *basechain;
496 if (!nft_is_base_chain(chain))
499 basechain = nft_base_chain(chain);
500 policy = ppolicy ? *ppolicy : basechain->policy;
502 /* Only default policy to accept is supported for now. */
503 if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
506 return nft_flow_block_chain(basechain, NULL, cmd);
509 static void nft_flow_rule_offload_abort(struct net *net,
510 struct nft_trans *trans)
512 struct nftables_pernet *nft_net = nft_pernet(net);
515 list_for_each_entry_continue_reverse(trans, &nft_net->commit_list, list) {
516 if (trans->ctx.family != NFPROTO_NETDEV)
519 switch (trans->msg_type) {
520 case NFT_MSG_NEWCHAIN:
521 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
522 nft_trans_chain_update(trans))
525 err = nft_flow_offload_chain(trans->ctx.chain, NULL,
528 case NFT_MSG_DELCHAIN:
529 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
532 err = nft_flow_offload_chain(trans->ctx.chain, NULL,
535 case NFT_MSG_NEWRULE:
536 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
539 err = nft_flow_offload_rule(trans->ctx.chain,
540 nft_trans_rule(trans),
541 NULL, FLOW_CLS_DESTROY);
543 case NFT_MSG_DELRULE:
544 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
547 err = nft_flow_offload_rule(trans->ctx.chain,
548 nft_trans_rule(trans),
549 nft_trans_flow_rule(trans),
554 if (WARN_ON_ONCE(err))
559 int nft_flow_rule_offload_commit(struct net *net)
561 struct nftables_pernet *nft_net = nft_pernet(net);
562 struct nft_trans *trans;
566 list_for_each_entry(trans, &nft_net->commit_list, list) {
567 if (trans->ctx.family != NFPROTO_NETDEV)
570 switch (trans->msg_type) {
571 case NFT_MSG_NEWCHAIN:
572 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
573 nft_trans_chain_update(trans))
576 policy = nft_trans_chain_policy(trans);
577 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
580 case NFT_MSG_DELCHAIN:
581 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
584 policy = nft_trans_chain_policy(trans);
585 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
588 case NFT_MSG_NEWRULE:
589 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
592 if (trans->ctx.flags & NLM_F_REPLACE ||
593 !(trans->ctx.flags & NLM_F_APPEND)) {
597 err = nft_flow_offload_rule(trans->ctx.chain,
598 nft_trans_rule(trans),
599 nft_trans_flow_rule(trans),
602 case NFT_MSG_DELRULE:
603 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
606 err = nft_flow_offload_rule(trans->ctx.chain,
607 nft_trans_rule(trans),
608 NULL, FLOW_CLS_DESTROY);
613 nft_flow_rule_offload_abort(net, trans);
621 static struct nft_chain *__nft_offload_get_chain(const struct nftables_pernet *nft_net,
622 struct net_device *dev)
624 struct nft_base_chain *basechain;
625 struct nft_hook *hook, *found;
626 const struct nft_table *table;
627 struct nft_chain *chain;
629 list_for_each_entry(table, &nft_net->tables, list) {
630 if (table->family != NFPROTO_NETDEV)
633 list_for_each_entry(chain, &table->chains, list) {
634 if (!nft_is_base_chain(chain) ||
635 !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
639 basechain = nft_base_chain(chain);
640 list_for_each_entry(hook, &basechain->hook_list, list) {
641 if (hook->ops.dev != dev)
657 static int nft_offload_netdev_event(struct notifier_block *this,
658 unsigned long event, void *ptr)
660 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
661 struct nftables_pernet *nft_net;
662 struct net *net = dev_net(dev);
663 struct nft_chain *chain;
665 if (event != NETDEV_UNREGISTER)
668 nft_net = nft_pernet(net);
669 mutex_lock(&nft_net->commit_mutex);
670 chain = __nft_offload_get_chain(nft_net, dev);
672 nft_flow_block_chain(nft_base_chain(chain), dev,
675 mutex_unlock(&nft_net->commit_mutex);
680 static struct notifier_block nft_offload_netdev_notifier = {
681 .notifier_call = nft_offload_netdev_event,
684 int nft_offload_init(void)
686 return register_netdevice_notifier(&nft_offload_netdev_notifier);
689 void nft_offload_exit(void)
691 unregister_netdevice_notifier(&nft_offload_netdev_notifier);