include/net/gro.h

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2
   3 #ifndef _NET_IPV6_GRO_H
   4 #define _NET_IPV6_GRO_H
   5
   6 #include <linux/indirect_call_wrapper.h>
   7 #include <linux/ip.h>
   8 #include <linux/ipv6.h>
   9 #include <net/ip6_checksum.h>
  10 #include <linux/skbuff.h>
  11 #include <net/udp.h>
  12
  13 struct napi_gro_cb {
  14         union {
  15                 struct {
  16                         /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
  17                         void    *frag0;
  18
  19                         /* Length of frag0. */
  20                         unsigned int frag0_len;
  21                 };
  22
  23                 struct {
  24                         /* used in skb_gro_receive() slow path */
  25                         struct sk_buff *last;
  26
  27                         /* jiffies when first packet was created/queued */
  28                         unsigned long age;
  29                 };
  30         };
  31
  32         /* This indicates where we are processing relative to skb->data. */
  33         int     data_offset;
  34
  35         /* This is non-zero if the packet cannot be merged with the new skb. */
  36         u16     flush;
  37
  38         /* Save the IP ID here and check when we get to the transport layer */
  39         u16     flush_id;
  40
  41         /* Number of segments aggregated. */
  42         u16     count;
  43
  44         /* Used in ipv6_gro_receive() and foo-over-udp and esp-in-udp */
  45         u16     proto;
  46
  47 /* Used in napi_gro_cb::free */
  48 #define NAPI_GRO_FREE             1
  49 #define NAPI_GRO_FREE_STOLEN_HEAD 2
  50         /* portion of the cb set to zero at every gro iteration */
  51         struct_group(zeroed,
  52
  53                 /* Start offset for remote checksum offload */
  54                 u16     gro_remcsum_start;
  55
  56                 /* This is non-zero if the packet may be of the same flow. */
  57                 u8      same_flow:1;
  58
  59                 /* Used in tunnel GRO receive */
  60                 u8      encap_mark:1;
  61
  62                 /* GRO checksum is valid */
  63                 u8      csum_valid:1;
  64
  65                 /* Number of checksums via CHECKSUM_UNNECESSARY */
  66                 u8      csum_cnt:3;
  67
  68                 /* Free the skb? */
  69                 u8      free:2;
  70
  71                 /* Used in foo-over-udp, set in udp[46]_gro_receive */
  72                 u8      is_ipv6:1;
  73
  74                 /* Used in GRE, set in fou/gue_gro_receive */
  75                 u8      is_fou:1;
  76
  77                 /* Used to determine if flush_id can be ignored */
  78                 u8      is_atomic:1;
  79
  80                 /* Number of gro_receive callbacks this packet already went through */
  81                 u8 recursion_counter:4;
  82
  83                 /* GRO is done by frag_list pointer chaining. */
  84                 u8      is_flist:1;
  85         );
  86
  87         /* used to support CHECKSUM_COMPLETE for tunneling protocols */
  88         __wsum  csum;
  89 };
  90
  91 #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
  92
  93 #define GRO_RECURSION_LIMIT 15
  94 static inline int gro_recursion_inc_test(struct sk_buff *skb)
  95 {
  96         return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
  97 }
  98
  99 typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);
 100 static inline struct sk_buff *call_gro_receive(gro_receive_t cb,
 101                                                struct list_head *head,
 102                                                struct sk_buff *skb)
 103 {
 104         if (unlikely(gro_recursion_inc_test(skb))) {
 105                 NAPI_GRO_CB(skb)->flush |= 1;
 106                 return NULL;
 107         }
 108
 109         return cb(head, skb);
 110 }
 111
 112 typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,
 113                                             struct sk_buff *);
 114 static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,
 115                                                   struct sock *sk,
 116                                                   struct list_head *head,
 117                                                   struct sk_buff *skb)
 118 {
 119         if (unlikely(gro_recursion_inc_test(skb))) {
 120                 NAPI_GRO_CB(skb)->flush |= 1;
 121                 return NULL;
 122         }
 123
 124         return cb(sk, head, skb);
 125 }
 126
 127 static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
 128 {
 129         return NAPI_GRO_CB(skb)->data_offset;
 130 }
 131
 132 static inline unsigned int skb_gro_len(const struct sk_buff *skb)
 133 {
 134         return skb->len - NAPI_GRO_CB(skb)->data_offset;
 135 }
 136
 137 static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
 138 {
 139         NAPI_GRO_CB(skb)->data_offset += len;
 140 }
 141
 142 static inline void *skb_gro_header_fast(struct sk_buff *skb,
 143                                         unsigned int offset)
 144 {
 145         return NAPI_GRO_CB(skb)->frag0 + offset;
 146 }
 147
 148 static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
 149 {
 150         return NAPI_GRO_CB(skb)->frag0_len < hlen;
 151 }
 152
 153 static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
 154 {
 155         NAPI_GRO_CB(skb)->frag0 = NULL;
 156         NAPI_GRO_CB(skb)->frag0_len = 0;
 157 }
 158
 159 static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
 160                                         unsigned int offset)
 161 {
 162         if (!pskb_may_pull(skb, hlen))
 163                 return NULL;
 164
 165         skb_gro_frag0_invalidate(skb);
 166         return skb->data + offset;
 167 }
 168
 169 static inline void *skb_gro_header(struct sk_buff *skb,
 170                                         unsigned int hlen, unsigned int offset)
 171 {
 172         void *ptr;
 173
 174         ptr = skb_gro_header_fast(skb, offset);
 175         if (skb_gro_header_hard(skb, hlen))
 176                 ptr = skb_gro_header_slow(skb, hlen, offset);
 177         return ptr;
 178 }
 179
 180 static inline void *skb_gro_network_header(struct sk_buff *skb)
 181 {
 182         return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
 183                skb_network_offset(skb);
 184 }
 185
 186 static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto)
 187 {
 188         const struct iphdr *iph = skb_gro_network_header(skb);
 189
 190         return csum_tcpudp_nofold(iph->saddr, iph->daddr,
 191                                   skb_gro_len(skb), proto, 0);
 192 }
 193
 194 static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
 195                                         const void *start, unsigned int len)
 196 {
 197         if (NAPI_GRO_CB(skb)->csum_valid)
 198                 NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len,
 199                                                 wsum_negate(NAPI_GRO_CB(skb)->csum)));
 200 }
 201
 202 /* GRO checksum functions. These are logical equivalents of the normal
 203  * checksum functions (in skbuff.h) except that they operate on the GRO
 204  * offsets and fields in sk_buff.
 205  */
 206
 207 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
 208
 209 static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
 210 {
 211         return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
 212 }
 213
 214 static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
 215                                                       bool zero_okay,
 216                                                       __sum16 check)
 217 {
 218         return ((skb->ip_summed != CHECKSUM_PARTIAL ||
 219                 skb_checksum_start_offset(skb) <
 220                  skb_gro_offset(skb)) &&
 221                 !skb_at_gro_remcsum_start(skb) &&
 222                 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
 223                 (!zero_okay || check));
 224 }
 225
 226 static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
 227                                                            __wsum psum)
 228 {
 229         if (NAPI_GRO_CB(skb)->csum_valid &&
 230             !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
 231                 return 0;
 232
 233         NAPI_GRO_CB(skb)->csum = psum;
 234
 235         return __skb_gro_checksum_complete(skb);
 236 }
 237
 238 static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
 239 {
 240         if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
 241                 /* Consume a checksum from CHECKSUM_UNNECESSARY */
 242                 NAPI_GRO_CB(skb)->csum_cnt--;
 243         } else {
 244                 /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we
 245                  * verified a new top level checksum or an encapsulated one
 246                  * during GRO. This saves work if we fallback to normal path.
 247                  */
 248                 __skb_incr_checksum_unnecessary(skb);
 249         }
 250 }
 251
 252 #define __skb_gro_checksum_validate(skb, proto, zero_okay, check,       \
 253                                     compute_pseudo)                     \
 254 ({                                                                      \
 255         __sum16 __ret = 0;                                              \
 256         if (__skb_gro_checksum_validate_needed(skb, zero_okay, check))  \
 257                 __ret = __skb_gro_checksum_validate_complete(skb,       \
 258                                 compute_pseudo(skb, proto));            \
 259         if (!__ret)                                                     \
 260                 skb_gro_incr_csum_unnecessary(skb);                     \
 261         __ret;                                                          \
 262 })
 263
 264 #define skb_gro_checksum_validate(skb, proto, compute_pseudo)           \
 265         __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
 266
 267 #define skb_gro_checksum_validate_zero_check(skb, proto, check,         \
 268                                              compute_pseudo)            \
 269         __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
 270
 271 #define skb_gro_checksum_simple_validate(skb)                           \
 272         __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
 273
 274 static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
 275 {
 276         return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
 277                 !NAPI_GRO_CB(skb)->csum_valid);
 278 }
 279
 280 static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
 281                                               __wsum pseudo)
 282 {
 283         NAPI_GRO_CB(skb)->csum = ~pseudo;
 284         NAPI_GRO_CB(skb)->csum_valid = 1;
 285 }
 286
 287 #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo)        \
 288 do {                                                                    \
 289         if (__skb_gro_checksum_convert_check(skb))                      \
 290                 __skb_gro_checksum_convert(skb,                         \
 291                                            compute_pseudo(skb, proto)); \
 292 } while (0)
 293
 294 struct gro_remcsum {
 295         int offset;
 296         __wsum delta;
 297 };
 298
 299 static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
 300 {
 301         grc->offset = 0;
 302         grc->delta = 0;
 303 }
 304
 305 static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
 306                                             unsigned int off, size_t hdrlen,
 307                                             int start, int offset,
 308                                             struct gro_remcsum *grc,
 309                                             bool nopartial)
 310 {
 311         __wsum delta;
 312         size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
 313
 314         BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
 315
 316         if (!nopartial) {
 317                 NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
 318                 return ptr;
 319         }
 320
 321         ptr = skb_gro_header(skb, off + plen, off);
 322         if (!ptr)
 323                 return NULL;
 324
 325         delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
 326                                start, offset);
 327
 328         /* Adjust skb->csum since we changed the packet */
 329         NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
 330
 331         grc->offset = off + hdrlen + offset;
 332         grc->delta = delta;
 333
 334         return ptr;
 335 }
 336
 337 static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
 338                                            struct gro_remcsum *grc)
 339 {
 340         void *ptr;
 341         size_t plen = grc->offset + sizeof(u16);
 342
 343         if (!grc->delta)
 344                 return;
 345
 346         ptr = skb_gro_header(skb, plen, grc->offset);
 347         if (!ptr)
 348                 return;
 349
 350         remcsum_unadjust((__sum16 *)ptr, grc->delta);
 351 }
 352
 353 #ifdef CONFIG_XFRM_OFFLOAD
 354 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
 355 {
 356         if (PTR_ERR(pp) != -EINPROGRESS)
 357                 NAPI_GRO_CB(skb)->flush |= flush;
 358 }
 359 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
 360                                                struct sk_buff *pp,
 361                                                int flush,
 362                                                struct gro_remcsum *grc)
 363 {
 364         if (PTR_ERR(pp) != -EINPROGRESS) {
 365                 NAPI_GRO_CB(skb)->flush |= flush;
 366                 skb_gro_remcsum_cleanup(skb, grc);
 367                 skb->remcsum_offload = 0;
 368         }
 369 }
 370 #else
 371 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
 372 {
 373         NAPI_GRO_CB(skb)->flush |= flush;
 374 }
 375 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
 376                                                struct sk_buff *pp,
 377                                                int flush,
 378                                                struct gro_remcsum *grc)
 379 {
 380         NAPI_GRO_CB(skb)->flush |= flush;
 381         skb_gro_remcsum_cleanup(skb, grc);
 382         skb->remcsum_offload = 0;
 383 }
 384 #endif
 385
 386 INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,
 387                                                            struct sk_buff *));
 388 INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
 389 INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
 390                                                            struct sk_buff *));
 391 INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
 392
 393 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *,
 394                                                            struct sk_buff *));
 395 INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int));
 396
 397 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *,
 398                                                            struct sk_buff *));
 399 INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int));
 400
 401 #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb)   \
 402 ({                                                              \
 403         unlikely(gro_recursion_inc_test(skb)) ?                 \
 404                 NAPI_GRO_CB(skb)->flush |= 1, NULL :            \
 405                 INDIRECT_CALL_INET(cb, f2, f1, head, skb);      \
 406 })
 407
 408 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
 409                                 struct udphdr *uh, struct sock *sk);
 410 int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
 411
 412 static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
 413 {
 414         struct udphdr *uh;
 415         unsigned int hlen, off;
 416
 417         off  = skb_gro_offset(skb);
 418         hlen = off + sizeof(*uh);
 419         uh   = skb_gro_header(skb, hlen, off);
 420
 421         return uh;
 422 }
 423
 424 static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto)
 425 {
 426         const struct ipv6hdr *iph = skb_gro_network_header(skb);
 427
 428         return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr,
 429                                             skb_gro_len(skb), proto, 0));
 430 }
 431
 432 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
 433
 434 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
 435 static inline void gro_normal_list(struct napi_struct *napi)
 436 {
 437         if (!napi->rx_count)
 438                 return;
 439         netif_receive_skb_list_internal(&napi->rx_list);
 440         INIT_LIST_HEAD(&napi->rx_list);
 441         napi->rx_count = 0;
 442 }
 443
 444 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
 445  * pass the whole batch up to the stack.
 446  */
 447 static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
 448 {
 449         list_add_tail(&skb->list, &napi->rx_list);
 450         napi->rx_count += segs;
 451         if (napi->rx_count >= READ_ONCE(gro_normal_batch))
 452                 gro_normal_list(napi);
 453 }
 454
 455 /* This function is the alternative of 'inet_iif' and 'inet_sdif'
 456  * functions in case we can not rely on fields of IPCB.
 457  *
 458  * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized.
 459  * The caller must hold the RCU read lock.
 460  */
 461 static inline void inet_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif)
 462 {
 463         *iif = inet_iif(skb) ?: skb->dev->ifindex;
 464         *sdif = 0;
 465
 466 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
 467         if (netif_is_l3_slave(skb->dev)) {
 468                 struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev);
 469
 470                 *sdif = *iif;
 471                 *iif = master ? master->ifindex : 0;
 472         }
 473 #endif
 474 }
 475
 476 /* This function is the alternative of 'inet6_iif' and 'inet6_sdif'
 477  * functions in case we can not rely on fields of IP6CB.
 478  *
 479  * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized.
 480  * The caller must hold the RCU read lock.
 481  */
 482 static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif)
 483 {
 484         /* using skb->dev->ifindex because skb_dst(skb) is not initialized */
 485         *iif = skb->dev->ifindex;
 486         *sdif = 0;
 487
 488 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
 489         if (netif_is_l3_slave(skb->dev)) {
 490                 struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev);
 491
 492                 *sdif = *iif;
 493                 *iif = master ? master->ifindex : 0;
 494         }
 495 #endif
 496 }
 497
 498 extern struct list_head offload_base;
 499
 500 #endif /* _NET_IPV6_GRO_H */