net/xfrm/xfrm_device.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * xfrm_device.c - IPsec device offloading code.
   4  *
   5  * Copyright (c) 2015 secunet Security Networks AG
   6  *
   7  * Author:
   8  * Steffen Klassert <steffen.klassert@secunet.com>
   9  */
  10
  11 #include <linux/errno.h>
  12 #include <linux/module.h>
  13 #include <linux/netdevice.h>
  14 #include <linux/skbuff.h>
  15 #include <linux/slab.h>
  16 #include <linux/spinlock.h>
  17 #include <net/dst.h>
  18 #include <net/xfrm.h>
  19 #include <linux/notifier.h>
  20
  21 #ifdef CONFIG_XFRM_OFFLOAD
  22 static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb,
  23                                   unsigned int hsize)
  24 {
  25         struct xfrm_offload *xo = xfrm_offload(skb);
  26
  27         skb_reset_mac_len(skb);
  28         if (xo->flags & XFRM_GSO_SEGMENT)
  29                 skb->transport_header -= x->props.header_len;
  30
  31         pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);
  32 }
  33
  34 static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb,
  35                                     unsigned int hsize)
  36
  37 {
  38         struct xfrm_offload *xo = xfrm_offload(skb);
  39
  40         if (xo->flags & XFRM_GSO_SEGMENT)
  41                 skb->transport_header = skb->network_header + hsize;
  42
  43         skb_reset_mac_len(skb);
  44         pskb_pull(skb, skb->mac_len + x->props.header_len);
  45 }
  46
  47 /* Adjust pointers into the packet when IPsec is done at layer2 */
  48 static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb)
  49 {
  50         switch (x->outer_mode.encap) {
  51         case XFRM_MODE_TUNNEL:
  52                 if (x->outer_mode.family == AF_INET)
  53                         return __xfrm_mode_tunnel_prep(x, skb,
  54                                                        sizeof(struct iphdr));
  55                 if (x->outer_mode.family == AF_INET6)
  56                         return __xfrm_mode_tunnel_prep(x, skb,
  57                                                        sizeof(struct ipv6hdr));
  58                 break;
  59         case XFRM_MODE_TRANSPORT:
  60                 if (x->outer_mode.family == AF_INET)
  61                         return __xfrm_transport_prep(x, skb,
  62                                                      sizeof(struct iphdr));
  63                 if (x->outer_mode.family == AF_INET6)
  64                         return __xfrm_transport_prep(x, skb,
  65                                                      sizeof(struct ipv6hdr));
  66                 break;
  67         case XFRM_MODE_ROUTEOPTIMIZATION:
  68         case XFRM_MODE_IN_TRIGGER:
  69         case XFRM_MODE_BEET:
  70                 break;
  71         }
  72 }
  73
  74 struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
  75 {
  76         int err;
  77         unsigned long flags;
  78         struct xfrm_state *x;
  79         struct sk_buff *skb2;
  80         struct softnet_data *sd;
  81         netdev_features_t esp_features = features;
  82         struct xfrm_offload *xo = xfrm_offload(skb);
  83         struct sec_path *sp;
  84
  85         if (!xo || (xo->flags & XFRM_XMIT))
  86                 return skb;
  87
  88         if (!(features & NETIF_F_HW_ESP))
  89                 esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);
  90
  91         sp = skb_sec_path(skb);
  92         x = sp->xvec[sp->len - 1];
  93         if (xo->flags & XFRM_GRO || x->xso.flags & XFRM_OFFLOAD_INBOUND)
  94                 return skb;
  95
  96         local_irq_save(flags);
  97         sd = this_cpu_ptr(&softnet_data);
  98         err = !skb_queue_empty(&sd->xfrm_backlog);
  99         local_irq_restore(flags);
 100
 101         if (err) {
 102                 *again = true;
 103                 return skb;
 104         }
 105
 106         xo->flags |= XFRM_XMIT;
 107
 108         if (skb_is_gso(skb)) {
 109                 struct net_device *dev = skb->dev;
 110
 111                 if (unlikely(x->xso.dev != dev)) {
 112                         struct sk_buff *segs;
 113
 114                         /* Packet got rerouted, fixup features and segment it. */
 115                         esp_features = esp_features & ~(NETIF_F_HW_ESP
 116                                                         | NETIF_F_GSO_ESP);
 117
 118                         segs = skb_gso_segment(skb, esp_features);
 119                         if (IS_ERR(segs)) {
 120                                 kfree_skb(skb);
 121                                 atomic_long_inc(&dev->tx_dropped);
 122                                 return NULL;
 123                         } else {
 124                                 consume_skb(skb);
 125                                 skb = segs;
 126                         }
 127                 }
 128         }
 129
 130         if (!skb->next) {
 131                 esp_features |= skb->dev->gso_partial_features;
 132                 xfrm_outer_mode_prep(x, skb);
 133
 134                 xo->flags |= XFRM_DEV_RESUME;
 135
 136                 err = x->type_offload->xmit(x, skb, esp_features);
 137                 if (err) {
 138                         if (err == -EINPROGRESS)
 139                                 return NULL;
 140
 141                         XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
 142                         kfree_skb(skb);
 143                         return NULL;
 144                 }
 145
 146                 skb_push(skb, skb->data - skb_mac_header(skb));
 147
 148                 return skb;
 149         }
 150
 151         skb2 = skb;
 152
 153         do {
 154                 struct sk_buff *nskb = skb2->next;
 155
 156                 esp_features |= skb->dev->gso_partial_features;
 157                 skb_mark_not_on_list(skb2);
 158
 159                 xo = xfrm_offload(skb2);
 160                 xo->flags |= XFRM_DEV_RESUME;
 161
 162                 xfrm_outer_mode_prep(x, skb2);
 163
 164                 err = x->type_offload->xmit(x, skb2, esp_features);
 165                 if (!err) {
 166                         skb2->next = nskb;
 167                 } else if (err != -EINPROGRESS) {
 168                         XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
 169                         skb2->next = nskb;
 170                         kfree_skb_list(skb2);
 171                         return NULL;
 172                 } else {
 173                         if (skb == skb2)
 174                                 skb = nskb;
 175
 176                         if (!skb)
 177                                 return NULL;
 178
 179                         goto skip_push;
 180                 }
 181
 182                 skb_push(skb2, skb2->data - skb_mac_header(skb2));
 183
 184 skip_push:
 185                 skb2 = nskb;
 186         } while (skb2);
 187
 188         return skb;
 189 }
 190 EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
 191
 192 int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
 193                        struct xfrm_user_offload *xuo)
 194 {
 195         int err;
 196         struct dst_entry *dst;
 197         struct net_device *dev;
 198         struct xfrm_state_offload *xso = &x->xso;
 199         xfrm_address_t *saddr;
 200         xfrm_address_t *daddr;
 201
 202         if (!x->type_offload)
 203                 return -EINVAL;
 204
 205         /* We don't yet support UDP encapsulation and TFC padding. */
 206         if (x->encap || x->tfcpad)
 207                 return -EINVAL;
 208
 209         if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND))
 210                 return -EINVAL;
 211
 212         dev = dev_get_by_index(net, xuo->ifindex);
 213         if (!dev) {
 214                 if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
 215                         saddr = &x->props.saddr;
 216                         daddr = &x->id.daddr;
 217                 } else {
 218                         saddr = &x->id.daddr;
 219                         daddr = &x->props.saddr;
 220                 }
 221
 222                 dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr,
 223                                         x->props.family,
 224                                         xfrm_smark_get(0, x));
 225                 if (IS_ERR(dst))
 226                         return 0;
 227
 228                 dev = dst->dev;
 229
 230                 dev_hold(dev);
 231                 dst_release(dst);
 232         }
 233
 234         if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
 235                 xso->dev = NULL;
 236                 dev_put(dev);
 237                 return 0;
 238         }
 239
 240         if (x->props.flags & XFRM_STATE_ESN &&
 241             !dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
 242                 xso->dev = NULL;
 243                 dev_put(dev);
 244                 return -EINVAL;
 245         }
 246
 247         xso->dev = dev;
 248         xso->num_exthdrs = 1;
 249         /* Don't forward bit that is not implemented */
 250         xso->flags = xuo->flags & ~XFRM_OFFLOAD_IPV6;
 251
 252         err = dev->xfrmdev_ops->xdo_dev_state_add(x);
 253         if (err) {
 254                 xso->num_exthdrs = 0;
 255                 xso->flags = 0;
 256                 xso->dev = NULL;
 257                 dev_put(dev);
 258
 259                 if (err != -EOPNOTSUPP)
 260                         return err;
 261         }
 262
 263         return 0;
 264 }
 265 EXPORT_SYMBOL_GPL(xfrm_dev_state_add);
 266
 267 bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
 268 {
 269         int mtu;
 270         struct dst_entry *dst = skb_dst(skb);
 271         struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
 272         struct net_device *dev = x->xso.dev;
 273
 274         if (!x->type_offload || x->encap)
 275                 return false;
 276
 277         if ((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
 278             (!xdst->child->xfrm)) {
 279                 mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
 280                 if (skb->len <= mtu)
 281                         goto ok;
 282
 283                 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
 284                         goto ok;
 285         }
 286
 287         return false;
 288
 289 ok:
 290         if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)
 291                 return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
 292
 293         return true;
 294 }
 295 EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);
 296
 297 void xfrm_dev_resume(struct sk_buff *skb)
 298 {
 299         struct net_device *dev = skb->dev;
 300         int ret = NETDEV_TX_BUSY;
 301         struct netdev_queue *txq;
 302         struct softnet_data *sd;
 303         unsigned long flags;
 304
 305         rcu_read_lock();
 306         txq = netdev_core_pick_tx(dev, skb, NULL);
 307
 308         HARD_TX_LOCK(dev, txq, smp_processor_id());
 309         if (!netif_xmit_frozen_or_stopped(txq))
 310                 skb = dev_hard_start_xmit(skb, dev, txq, &ret);
 311         HARD_TX_UNLOCK(dev, txq);
 312
 313         if (!dev_xmit_complete(ret)) {
 314                 local_irq_save(flags);
 315                 sd = this_cpu_ptr(&softnet_data);
 316                 skb_queue_tail(&sd->xfrm_backlog, skb);
 317                 raise_softirq_irqoff(NET_TX_SOFTIRQ);
 318                 local_irq_restore(flags);
 319         }
 320         rcu_read_unlock();
 321 }
 322 EXPORT_SYMBOL_GPL(xfrm_dev_resume);
 323
 324 void xfrm_dev_backlog(struct softnet_data *sd)
 325 {
 326         struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;
 327         struct sk_buff_head list;
 328         struct sk_buff *skb;
 329
 330         if (skb_queue_empty(xfrm_backlog))
 331                 return;
 332
 333         __skb_queue_head_init(&list);
 334
 335         spin_lock(&xfrm_backlog->lock);
 336         skb_queue_splice_init(xfrm_backlog, &list);
 337         spin_unlock(&xfrm_backlog->lock);
 338
 339         while (!skb_queue_empty(&list)) {
 340                 skb = __skb_dequeue(&list);
 341                 xfrm_dev_resume(skb);
 342         }
 343
 344 }
 345 #endif
 346
 347 static int xfrm_api_check(struct net_device *dev)
 348 {
 349 #ifdef CONFIG_XFRM_OFFLOAD
 350         if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
 351             !(dev->features & NETIF_F_HW_ESP))
 352                 return NOTIFY_BAD;
 353
 354         if ((dev->features & NETIF_F_HW_ESP) &&
 355             (!(dev->xfrmdev_ops &&
 356                dev->xfrmdev_ops->xdo_dev_state_add &&
 357                dev->xfrmdev_ops->xdo_dev_state_delete)))
 358                 return NOTIFY_BAD;
 359 #else
 360         if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))
 361                 return NOTIFY_BAD;
 362 #endif
 363
 364         return NOTIFY_DONE;
 365 }
 366
 367 static int xfrm_dev_register(struct net_device *dev)
 368 {
 369         return xfrm_api_check(dev);
 370 }
 371
 372 static int xfrm_dev_feat_change(struct net_device *dev)
 373 {
 374         return xfrm_api_check(dev);
 375 }
 376
 377 static int xfrm_dev_down(struct net_device *dev)
 378 {
 379         if (dev->features & NETIF_F_HW_ESP)
 380                 xfrm_dev_state_flush(dev_net(dev), dev, true);
 381
 382         return NOTIFY_DONE;
 383 }
 384
 385 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
 386 {
 387         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 388
 389         switch (event) {
 390         case NETDEV_REGISTER:
 391                 return xfrm_dev_register(dev);
 392
 393         case NETDEV_FEAT_CHANGE:
 394                 return xfrm_dev_feat_change(dev);
 395
 396         case NETDEV_DOWN:
 397         case NETDEV_UNREGISTER:
 398                 return xfrm_dev_down(dev);
 399         }
 400         return NOTIFY_DONE;
 401 }
 402
 403 static struct notifier_block xfrm_dev_notifier = {
 404         .notifier_call  = xfrm_dev_event,
 405 };
 406
 407 void __init xfrm_dev_init(void)
 408 {
 409         register_netdevice_notifier(&xfrm_dev_notifier);
 410 }