1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
4 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org>
6 * Development of this code funded by Astaro AG (http://www.astaro.com/)
9 #include <linux/kernel.h>
10 #include <linux/if_vlan.h>
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/netlink.h>
14 #include <linux/netfilter.h>
15 #include <linux/netfilter/nf_tables.h>
16 #include <net/netfilter/nf_tables_core.h>
17 #include <net/netfilter/nf_tables.h>
18 #include <net/netfilter/nf_tables_offload.h>
19 /* For layer 4 checksum field offset. */
20 #include <linux/tcp.h>
21 #include <linux/udp.h>
23 #include <linux/icmpv6.h>
25 #include <linux/ipv6.h>
26 #include <net/sctp/checksum.h>
28 static bool nft_payload_rebuild_vlan_hdr(const struct sk_buff *skb, int mac_off,
29 struct vlan_ethhdr *veth)
31 if (skb_copy_bits(skb, mac_off, veth, ETH_HLEN))
34 veth->h_vlan_proto = skb->vlan_proto;
35 veth->h_vlan_TCI = htons(skb_vlan_tag_get(skb));
36 veth->h_vlan_encapsulated_proto = skb->protocol;
41 /* add vlan header into the user buffer for if tag was removed by offloads */
43 nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u8 offset, u8 len)
45 int mac_off = skb_mac_header(skb) - skb->data;
46 u8 *vlanh, *dst_u8 = (u8 *) d;
47 struct vlan_ethhdr veth;
50 if ((skb->protocol == htons(ETH_P_8021AD) ||
51 skb->protocol == htons(ETH_P_8021Q)) &&
52 offset >= VLAN_ETH_HLEN && offset < VLAN_ETH_HLEN + VLAN_HLEN)
53 vlan_hlen += VLAN_HLEN;
56 if (offset < VLAN_ETH_HLEN + vlan_hlen) {
60 skb_copy_bits(skb, mac_off, &veth, VLAN_ETH_HLEN) < 0)
62 else if (!nft_payload_rebuild_vlan_hdr(skb, mac_off, &veth))
65 if (offset + len > VLAN_ETH_HLEN + vlan_hlen)
66 ethlen -= offset + len - VLAN_ETH_HLEN - vlan_hlen;
68 memcpy(dst_u8, vlanh + offset - vlan_hlen, ethlen);
75 offset = ETH_HLEN + vlan_hlen;
77 offset -= VLAN_HLEN + vlan_hlen;
80 return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0;
83 static int __nft_payload_inner_offset(struct nft_pktinfo *pkt)
85 unsigned int thoff = nft_thoff(pkt);
87 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
92 pkt->inneroff = thoff + sizeof(struct udphdr);
95 struct tcphdr *th, _tcph;
97 th = skb_header_pointer(pkt->skb, thoff, sizeof(_tcph), &_tcph);
101 pkt->inneroff = thoff + __tcp_hdrlen(th);
105 u32 offset = sizeof(struct gre_base_hdr);
106 struct gre_base_hdr *gre, _gre;
109 gre = skb_header_pointer(pkt->skb, thoff, sizeof(_gre), &_gre);
113 version = gre->flags & GRE_VERSION;
116 if (gre->flags & GRE_ROUTING)
119 if (gre->flags & GRE_CSUM) {
120 offset += sizeof_field(struct gre_full_hdr, csum) +
121 sizeof_field(struct gre_full_hdr, reserved1);
123 if (gre->flags & GRE_KEY)
124 offset += sizeof_field(struct gre_full_hdr, key);
126 if (gre->flags & GRE_SEQ)
127 offset += sizeof_field(struct gre_full_hdr, seq);
133 pkt->inneroff = thoff + offset;
137 pkt->inneroff = thoff;
143 pkt->flags |= NFT_PKTINFO_INNER;
148 int nft_payload_inner_offset(const struct nft_pktinfo *pkt)
150 if (!(pkt->flags & NFT_PKTINFO_INNER) &&
151 __nft_payload_inner_offset((struct nft_pktinfo *)pkt) < 0)
154 return pkt->inneroff;
157 static bool nft_payload_need_vlan_copy(const struct nft_payload *priv)
159 unsigned int len = priv->offset + priv->len;
161 /* data past ether src/dst requested, copy needed */
162 if (len > offsetof(struct ethhdr, h_proto))
168 void nft_payload_eval(const struct nft_expr *expr,
169 struct nft_regs *regs,
170 const struct nft_pktinfo *pkt)
172 const struct nft_payload *priv = nft_expr_priv(expr);
173 const struct sk_buff *skb = pkt->skb;
174 u32 *dest = ®s->data[priv->dreg];
177 if (priv->len % NFT_REG32_SIZE)
178 dest[priv->len / NFT_REG32_SIZE] = 0;
180 switch (priv->base) {
181 case NFT_PAYLOAD_LL_HEADER:
182 if (!skb_mac_header_was_set(skb) || skb_mac_header_len(skb) == 0)
185 if (skb_vlan_tag_present(skb) &&
186 nft_payload_need_vlan_copy(priv)) {
187 if (!nft_payload_copy_vlan(dest, skb,
188 priv->offset, priv->len))
192 offset = skb_mac_header(skb) - skb->data;
194 case NFT_PAYLOAD_NETWORK_HEADER:
195 offset = skb_network_offset(skb);
197 case NFT_PAYLOAD_TRANSPORT_HEADER:
198 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
200 offset = nft_thoff(pkt);
202 case NFT_PAYLOAD_INNER_HEADER:
203 offset = nft_payload_inner_offset(pkt);
211 offset += priv->offset;
213 if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
217 regs->verdict.code = NFT_BREAK;
220 static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = {
221 [NFTA_PAYLOAD_SREG] = { .type = NLA_U32 },
222 [NFTA_PAYLOAD_DREG] = { .type = NLA_U32 },
223 [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 },
224 [NFTA_PAYLOAD_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255),
225 [NFTA_PAYLOAD_LEN] = NLA_POLICY_MAX(NLA_BE32, 255),
226 [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 },
227 [NFTA_PAYLOAD_CSUM_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255),
228 [NFTA_PAYLOAD_CSUM_FLAGS] = { .type = NLA_U32 },
231 static int nft_payload_init(const struct nft_ctx *ctx,
232 const struct nft_expr *expr,
233 const struct nlattr * const tb[])
235 struct nft_payload *priv = nft_expr_priv(expr);
237 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
238 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
239 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
241 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG],
242 &priv->dreg, NULL, NFT_DATA_VALUE,
246 static int nft_payload_dump(struct sk_buff *skb,
247 const struct nft_expr *expr, bool reset)
249 const struct nft_payload *priv = nft_expr_priv(expr);
251 if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) ||
252 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
253 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
254 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)))
255 goto nla_put_failure;
262 static bool nft_payload_reduce(struct nft_regs_track *track,
263 const struct nft_expr *expr)
265 const struct nft_payload *priv = nft_expr_priv(expr);
266 const struct nft_payload *payload;
268 if (!nft_reg_track_cmp(track, expr, priv->dreg)) {
269 nft_reg_track_update(track, expr, priv->dreg, priv->len);
273 payload = nft_expr_priv(track->regs[priv->dreg].selector);
274 if (priv->base != payload->base ||
275 priv->offset != payload->offset ||
276 priv->len != payload->len) {
277 nft_reg_track_update(track, expr, priv->dreg, priv->len);
281 if (!track->regs[priv->dreg].bitwise)
284 return nft_expr_reduce_bitwise(track, expr);
287 static bool nft_payload_offload_mask(struct nft_offload_reg *reg,
288 u32 priv_len, u32 field_len)
290 unsigned int remainder, delta, k;
291 struct nft_data mask = {};
292 __be32 remainder_mask;
294 if (priv_len == field_len) {
295 memset(®->mask, 0xff, priv_len);
297 } else if (priv_len > field_len) {
301 memset(&mask, 0xff, field_len);
302 remainder = priv_len % sizeof(u32);
304 k = priv_len / sizeof(u32);
305 delta = field_len - priv_len;
306 remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1));
307 mask.data[k] = (__force u32)remainder_mask;
310 memcpy(®->mask, &mask, field_len);
315 static int nft_payload_offload_ll(struct nft_offload_ctx *ctx,
316 struct nft_flow_rule *flow,
317 const struct nft_payload *priv)
319 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
321 switch (priv->offset) {
322 case offsetof(struct ethhdr, h_source):
323 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
326 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
329 case offsetof(struct ethhdr, h_dest):
330 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
333 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
336 case offsetof(struct ethhdr, h_proto):
337 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
340 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic,
341 n_proto, sizeof(__be16), reg);
342 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
344 case offsetof(struct vlan_ethhdr, h_vlan_TCI):
345 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
348 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_VLAN, vlan,
349 vlan_tci, sizeof(__be16), reg,
350 NFT_OFFLOAD_F_NETWORK2HOST);
352 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto):
353 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
356 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_VLAN, vlan,
357 vlan_tpid, sizeof(__be16), reg);
358 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
360 case offsetof(struct vlan_ethhdr, h_vlan_TCI) + sizeof(struct vlan_hdr):
361 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
364 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
365 vlan_tci, sizeof(__be16), reg,
366 NFT_OFFLOAD_F_NETWORK2HOST);
368 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto) +
369 sizeof(struct vlan_hdr):
370 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
373 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
374 vlan_tpid, sizeof(__be16), reg);
375 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
384 static int nft_payload_offload_ip(struct nft_offload_ctx *ctx,
385 struct nft_flow_rule *flow,
386 const struct nft_payload *priv)
388 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
390 switch (priv->offset) {
391 case offsetof(struct iphdr, saddr):
392 if (!nft_payload_offload_mask(reg, priv->len,
393 sizeof(struct in_addr)))
396 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src,
397 sizeof(struct in_addr), reg);
398 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
400 case offsetof(struct iphdr, daddr):
401 if (!nft_payload_offload_mask(reg, priv->len,
402 sizeof(struct in_addr)))
405 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst,
406 sizeof(struct in_addr), reg);
407 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
409 case offsetof(struct iphdr, protocol):
410 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
413 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
415 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
424 static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx,
425 struct nft_flow_rule *flow,
426 const struct nft_payload *priv)
428 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
430 switch (priv->offset) {
431 case offsetof(struct ipv6hdr, saddr):
432 if (!nft_payload_offload_mask(reg, priv->len,
433 sizeof(struct in6_addr)))
436 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src,
437 sizeof(struct in6_addr), reg);
438 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
440 case offsetof(struct ipv6hdr, daddr):
441 if (!nft_payload_offload_mask(reg, priv->len,
442 sizeof(struct in6_addr)))
445 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst,
446 sizeof(struct in6_addr), reg);
447 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
449 case offsetof(struct ipv6hdr, nexthdr):
450 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
453 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
455 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
464 static int nft_payload_offload_nh(struct nft_offload_ctx *ctx,
465 struct nft_flow_rule *flow,
466 const struct nft_payload *priv)
470 switch (ctx->dep.l3num) {
471 case htons(ETH_P_IP):
472 err = nft_payload_offload_ip(ctx, flow, priv);
474 case htons(ETH_P_IPV6):
475 err = nft_payload_offload_ip6(ctx, flow, priv);
484 static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx,
485 struct nft_flow_rule *flow,
486 const struct nft_payload *priv)
488 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
490 switch (priv->offset) {
491 case offsetof(struct tcphdr, source):
492 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
495 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
496 sizeof(__be16), reg);
498 case offsetof(struct tcphdr, dest):
499 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
502 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
503 sizeof(__be16), reg);
512 static int nft_payload_offload_udp(struct nft_offload_ctx *ctx,
513 struct nft_flow_rule *flow,
514 const struct nft_payload *priv)
516 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
518 switch (priv->offset) {
519 case offsetof(struct udphdr, source):
520 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
523 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
524 sizeof(__be16), reg);
526 case offsetof(struct udphdr, dest):
527 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
530 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
531 sizeof(__be16), reg);
540 static int nft_payload_offload_th(struct nft_offload_ctx *ctx,
541 struct nft_flow_rule *flow,
542 const struct nft_payload *priv)
546 switch (ctx->dep.protonum) {
548 err = nft_payload_offload_tcp(ctx, flow, priv);
551 err = nft_payload_offload_udp(ctx, flow, priv);
560 static int nft_payload_offload(struct nft_offload_ctx *ctx,
561 struct nft_flow_rule *flow,
562 const struct nft_expr *expr)
564 const struct nft_payload *priv = nft_expr_priv(expr);
567 switch (priv->base) {
568 case NFT_PAYLOAD_LL_HEADER:
569 err = nft_payload_offload_ll(ctx, flow, priv);
571 case NFT_PAYLOAD_NETWORK_HEADER:
572 err = nft_payload_offload_nh(ctx, flow, priv);
574 case NFT_PAYLOAD_TRANSPORT_HEADER:
575 err = nft_payload_offload_th(ctx, flow, priv);
584 static const struct nft_expr_ops nft_payload_ops = {
585 .type = &nft_payload_type,
586 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
587 .eval = nft_payload_eval,
588 .init = nft_payload_init,
589 .dump = nft_payload_dump,
590 .reduce = nft_payload_reduce,
591 .offload = nft_payload_offload,
594 const struct nft_expr_ops nft_payload_fast_ops = {
595 .type = &nft_payload_type,
596 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
597 .eval = nft_payload_eval,
598 .init = nft_payload_init,
599 .dump = nft_payload_dump,
600 .reduce = nft_payload_reduce,
601 .offload = nft_payload_offload,
604 void nft_payload_inner_eval(const struct nft_expr *expr, struct nft_regs *regs,
605 const struct nft_pktinfo *pkt,
606 struct nft_inner_tun_ctx *tun_ctx)
608 const struct nft_payload *priv = nft_expr_priv(expr);
609 const struct sk_buff *skb = pkt->skb;
610 u32 *dest = ®s->data[priv->dreg];
613 if (priv->len % NFT_REG32_SIZE)
614 dest[priv->len / NFT_REG32_SIZE] = 0;
616 switch (priv->base) {
617 case NFT_PAYLOAD_TUN_HEADER:
618 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TUN))
621 offset = tun_ctx->inner_tunoff;
623 case NFT_PAYLOAD_LL_HEADER:
624 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_LL))
627 offset = tun_ctx->inner_lloff;
629 case NFT_PAYLOAD_NETWORK_HEADER:
630 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_NH))
633 offset = tun_ctx->inner_nhoff;
635 case NFT_PAYLOAD_TRANSPORT_HEADER:
636 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TH))
639 offset = tun_ctx->inner_thoff;
645 offset += priv->offset;
647 if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
652 regs->verdict.code = NFT_BREAK;
655 static int nft_payload_inner_init(const struct nft_ctx *ctx,
656 const struct nft_expr *expr,
657 const struct nlattr * const tb[])
659 struct nft_payload *priv = nft_expr_priv(expr);
662 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
664 case NFT_PAYLOAD_TUN_HEADER:
665 case NFT_PAYLOAD_LL_HEADER:
666 case NFT_PAYLOAD_NETWORK_HEADER:
667 case NFT_PAYLOAD_TRANSPORT_HEADER:
674 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
675 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
677 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG],
678 &priv->dreg, NULL, NFT_DATA_VALUE,
682 static const struct nft_expr_ops nft_payload_inner_ops = {
683 .type = &nft_payload_type,
684 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
685 .init = nft_payload_inner_init,
686 .dump = nft_payload_dump,
687 /* direct call to nft_payload_inner_eval(). */
690 static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)
692 *sum = csum_fold(csum_add(csum_sub(~csum_unfold(*sum), fsum), tsum));
694 *sum = CSUM_MANGLED_0;
697 static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff)
699 struct udphdr *uh, _uh;
701 uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh);
705 return (__force bool)uh->check;
708 static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt,
710 unsigned int *l4csum_offset)
715 switch (pkt->tprot) {
717 *l4csum_offset = offsetof(struct tcphdr, check);
720 if (!nft_payload_udp_checksum(skb, nft_thoff(pkt)))
723 case IPPROTO_UDPLITE:
724 *l4csum_offset = offsetof(struct udphdr, check);
727 *l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
733 *l4csum_offset += nft_thoff(pkt);
737 static int nft_payload_csum_sctp(struct sk_buff *skb, int offset)
741 if (skb_ensure_writable(skb, offset + sizeof(*sh)))
744 sh = (struct sctphdr *)(skb->data + offset);
745 sh->checksum = sctp_compute_cksum(skb, offset);
746 skb->ip_summed = CHECKSUM_UNNECESSARY;
750 static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt,
752 __wsum fsum, __wsum tsum)
757 /* If we cannot determine layer 4 checksum offset or this packet doesn't
758 * require layer 4 checksum recalculation, skip this packet.
760 if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0)
763 if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
766 /* Checksum mangling for an arbitrary amount of bytes, based on
767 * inet_proto_csum_replace*() functions.
769 if (skb->ip_summed != CHECKSUM_PARTIAL) {
770 nft_csum_replace(&sum, fsum, tsum);
771 if (skb->ip_summed == CHECKSUM_COMPLETE) {
772 skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum),
776 sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum),
780 if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) ||
781 skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
787 static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src,
788 __wsum fsum, __wsum tsum, int csum_offset)
792 if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
795 nft_csum_replace(&sum, fsum, tsum);
796 if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) ||
797 skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
803 struct nft_payload_set {
804 enum nft_payload_bases base:8;
813 static void nft_payload_set_eval(const struct nft_expr *expr,
814 struct nft_regs *regs,
815 const struct nft_pktinfo *pkt)
817 const struct nft_payload_set *priv = nft_expr_priv(expr);
818 struct sk_buff *skb = pkt->skb;
819 const u32 *src = ®s->data[priv->sreg];
820 int offset, csum_offset;
823 switch (priv->base) {
824 case NFT_PAYLOAD_LL_HEADER:
825 if (!skb_mac_header_was_set(skb))
827 offset = skb_mac_header(skb) - skb->data;
829 case NFT_PAYLOAD_NETWORK_HEADER:
830 offset = skb_network_offset(skb);
832 case NFT_PAYLOAD_TRANSPORT_HEADER:
833 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
835 offset = nft_thoff(pkt);
837 case NFT_PAYLOAD_INNER_HEADER:
838 offset = nft_payload_inner_offset(pkt);
847 csum_offset = offset + priv->csum_offset;
848 offset += priv->offset;
850 if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) &&
851 ((priv->base != NFT_PAYLOAD_TRANSPORT_HEADER &&
852 priv->base != NFT_PAYLOAD_INNER_HEADER) ||
853 skb->ip_summed != CHECKSUM_PARTIAL)) {
854 fsum = skb_checksum(skb, offset, priv->len, 0);
855 tsum = csum_partial(src, priv->len, 0);
857 if (priv->csum_type == NFT_PAYLOAD_CSUM_INET &&
858 nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset))
861 if (priv->csum_flags &&
862 nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0)
866 if (skb_ensure_writable(skb, max(offset + priv->len, 0)) ||
867 skb_store_bits(skb, offset, src, priv->len) < 0)
870 if (priv->csum_type == NFT_PAYLOAD_CSUM_SCTP &&
871 pkt->tprot == IPPROTO_SCTP &&
872 skb->ip_summed != CHECKSUM_PARTIAL) {
873 if (pkt->fragoff == 0 &&
874 nft_payload_csum_sctp(skb, nft_thoff(pkt)))
880 regs->verdict.code = NFT_BREAK;
883 static int nft_payload_set_init(const struct nft_ctx *ctx,
884 const struct nft_expr *expr,
885 const struct nlattr * const tb[])
887 struct nft_payload_set *priv = nft_expr_priv(expr);
888 u32 csum_offset, csum_type = NFT_PAYLOAD_CSUM_NONE;
891 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
892 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
893 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
895 if (tb[NFTA_PAYLOAD_CSUM_TYPE])
896 csum_type = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE]));
897 if (tb[NFTA_PAYLOAD_CSUM_OFFSET]) {
898 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_CSUM_OFFSET], U8_MAX,
903 priv->csum_offset = csum_offset;
905 if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) {
908 flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS]));
909 if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR)
912 priv->csum_flags = flags;
916 case NFT_PAYLOAD_CSUM_NONE:
917 case NFT_PAYLOAD_CSUM_INET:
919 case NFT_PAYLOAD_CSUM_SCTP:
920 if (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER)
923 if (priv->csum_offset != offsetof(struct sctphdr, checksum))
929 priv->csum_type = csum_type;
931 return nft_parse_register_load(tb[NFTA_PAYLOAD_SREG], &priv->sreg,
935 static int nft_payload_set_dump(struct sk_buff *skb,
936 const struct nft_expr *expr, bool reset)
938 const struct nft_payload_set *priv = nft_expr_priv(expr);
940 if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) ||
941 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
942 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
943 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) ||
944 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) ||
945 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET,
946 htonl(priv->csum_offset)) ||
947 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags)))
948 goto nla_put_failure;
955 static bool nft_payload_set_reduce(struct nft_regs_track *track,
956 const struct nft_expr *expr)
960 for (i = 0; i < NFT_REG32_NUM; i++) {
961 if (!track->regs[i].selector)
964 if (track->regs[i].selector->ops != &nft_payload_ops &&
965 track->regs[i].selector->ops != &nft_payload_fast_ops)
968 __nft_reg_track_cancel(track, i);
974 static const struct nft_expr_ops nft_payload_set_ops = {
975 .type = &nft_payload_type,
976 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)),
977 .eval = nft_payload_set_eval,
978 .init = nft_payload_set_init,
979 .dump = nft_payload_set_dump,
980 .reduce = nft_payload_set_reduce,
983 static const struct nft_expr_ops *
984 nft_payload_select_ops(const struct nft_ctx *ctx,
985 const struct nlattr * const tb[])
987 enum nft_payload_bases base;
988 unsigned int offset, len;
991 if (tb[NFTA_PAYLOAD_BASE] == NULL ||
992 tb[NFTA_PAYLOAD_OFFSET] == NULL ||
993 tb[NFTA_PAYLOAD_LEN] == NULL)
994 return ERR_PTR(-EINVAL);
996 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
998 case NFT_PAYLOAD_LL_HEADER:
999 case NFT_PAYLOAD_NETWORK_HEADER:
1000 case NFT_PAYLOAD_TRANSPORT_HEADER:
1001 case NFT_PAYLOAD_INNER_HEADER:
1004 return ERR_PTR(-EOPNOTSUPP);
1007 if (tb[NFTA_PAYLOAD_SREG] != NULL) {
1008 if (tb[NFTA_PAYLOAD_DREG] != NULL)
1009 return ERR_PTR(-EINVAL);
1010 return &nft_payload_set_ops;
1013 if (tb[NFTA_PAYLOAD_DREG] == NULL)
1014 return ERR_PTR(-EINVAL);
1016 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U8_MAX, &offset);
1018 return ERR_PTR(err);
1020 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_LEN], U8_MAX, &len);
1022 return ERR_PTR(err);
1024 if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) &&
1025 base != NFT_PAYLOAD_LL_HEADER && base != NFT_PAYLOAD_INNER_HEADER)
1026 return &nft_payload_fast_ops;
1028 return &nft_payload_ops;
1031 struct nft_expr_type nft_payload_type __read_mostly = {
1033 .select_ops = nft_payload_select_ops,
1034 .inner_ops = &nft_payload_inner_ops,
1035 .policy = nft_payload_policy,
1036 .maxattr = NFTA_PAYLOAD_MAX,
1037 .owner = THIS_MODULE,