net/ipv6/ila/ila_xlat.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/jhash.h>
   3 #include <linux/netfilter.h>
   4 #include <linux/rcupdate.h>
   5 #include <linux/rhashtable.h>
   6 #include <linux/vmalloc.h>
   7 #include <net/genetlink.h>
   8 #include <net/ila.h>
   9 #include <net/netns/generic.h>
  10 #include <uapi/linux/genetlink.h>
  11 #include "ila.h"
  12
  13 struct ila_xlat_params {
  14         struct ila_params ip;
  15         int ifindex;
  16 };
  17
  18 struct ila_map {
  19         struct ila_xlat_params xp;
  20         struct rhash_head node;
  21         struct ila_map __rcu *next;
  22         struct rcu_head rcu;
  23 };
  24
  25 static unsigned int ila_net_id;
  26
  27 struct ila_net {
  28         struct rhashtable rhash_table;
  29         spinlock_t *locks; /* Bucket locks for entry manipulation */
  30         unsigned int locks_mask;
  31         bool hooks_registered;
  32 };
  33
  34 #define LOCKS_PER_CPU 10
  35
  36 static int alloc_ila_locks(struct ila_net *ilan)
  37 {
  38         unsigned int i, size;
  39         unsigned int nr_pcpus = num_possible_cpus();
  40
  41         nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
  42         size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);
  43
  44         if (sizeof(spinlock_t) != 0) {
  45                 ilan->locks = kvmalloc(size * sizeof(spinlock_t), GFP_KERNEL);
  46                 if (!ilan->locks)
  47                         return -ENOMEM;
  48                 for (i = 0; i < size; i++)
  49                         spin_lock_init(&ilan->locks[i]);
  50         }
  51         ilan->locks_mask = size - 1;
  52
  53         return 0;
  54 }
  55
  56 static u32 hashrnd __read_mostly;
  57 static __always_inline void __ila_hash_secret_init(void)
  58 {
  59         net_get_random_once(&hashrnd, sizeof(hashrnd));
  60 }
  61
  62 static inline u32 ila_locator_hash(struct ila_locator loc)
  63 {
  64         u32 *v = (u32 *)loc.v32;
  65
  66         __ila_hash_secret_init();
  67         return jhash_2words(v[0], v[1], hashrnd);
  68 }
  69
  70 static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
  71                                        struct ila_locator loc)
  72 {
  73         return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
  74 }
  75
  76 static inline int ila_cmp_wildcards(struct ila_map *ila,
  77                                     struct ila_addr *iaddr, int ifindex)
  78 {
  79         return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
  80 }
  81
  82 static inline int ila_cmp_params(struct ila_map *ila,
  83                                  struct ila_xlat_params *xp)
  84 {
  85         return (ila->xp.ifindex != xp->ifindex);
  86 }
  87
  88 static int ila_cmpfn(struct rhashtable_compare_arg *arg,
  89                      const void *obj)
  90 {
  91         const struct ila_map *ila = obj;
  92
  93         return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key);
  94 }
  95
  96 static inline int ila_order(struct ila_map *ila)
  97 {
  98         int score = 0;
  99
 100         if (ila->xp.ifindex)
 101                 score += 1 << 1;
 102
 103         return score;
 104 }
 105
 106 static const struct rhashtable_params rht_params = {
 107         .nelem_hint = 1024,
 108         .head_offset = offsetof(struct ila_map, node),
 109         .key_offset = offsetof(struct ila_map, xp.ip.locator_match),
 110         .key_len = sizeof(u64), /* identifier */
 111         .max_size = 1048576,
 112         .min_size = 256,
 113         .automatic_shrinking = true,
 114         .obj_cmpfn = ila_cmpfn,
 115 };
 116
 117 static struct genl_family ila_nl_family;
 118
 119 static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
 120         [ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
 121         [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
 122         [ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
 123         [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
 124 };
 125
 126 static int parse_nl_config(struct genl_info *info,
 127                            struct ila_xlat_params *xp)
 128 {
 129         memset(xp, 0, sizeof(*xp));
 130
 131         if (info->attrs[ILA_ATTR_LOCATOR])
 132                 xp->ip.locator.v64 = (__force __be64)nla_get_u64(
 133                         info->attrs[ILA_ATTR_LOCATOR]);
 134
 135         if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
 136                 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
 137                         info->attrs[ILA_ATTR_LOCATOR_MATCH]);
 138
 139         if (info->attrs[ILA_ATTR_CSUM_MODE])
 140                 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);
 141
 142         if (info->attrs[ILA_ATTR_IFINDEX])
 143                 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);
 144
 145         return 0;
 146 }
 147
 148 /* Must be called with rcu readlock */
 149 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
 150                                                    int ifindex,
 151                                                    struct ila_net *ilan)
 152 {
 153         struct ila_map *ila;
 154
 155         ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
 156                                      rht_params);
 157         while (ila) {
 158                 if (!ila_cmp_wildcards(ila, iaddr, ifindex))
 159                         return ila;
 160                 ila = rcu_access_pointer(ila->next);
 161         }
 162
 163         return NULL;
 164 }
 165
 166 /* Must be called with rcu readlock */
 167 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
 168                                                    struct ila_net *ilan)
 169 {
 170         struct ila_map *ila;
 171
 172         ila = rhashtable_lookup_fast(&ilan->rhash_table,
 173                                      &xp->ip.locator_match,
 174                                      rht_params);
 175         while (ila) {
 176                 if (!ila_cmp_params(ila, xp))
 177                         return ila;
 178                 ila = rcu_access_pointer(ila->next);
 179         }
 180
 181         return NULL;
 182 }
 183
 184 static inline void ila_release(struct ila_map *ila)
 185 {
 186         kfree_rcu(ila, rcu);
 187 }
 188
 189 static void ila_free_cb(void *ptr, void *arg)
 190 {
 191         struct ila_map *ila = (struct ila_map *)ptr, *next;
 192
 193         /* Assume rcu_readlock held */
 194         while (ila) {
 195                 next = rcu_access_pointer(ila->next);
 196                 ila_release(ila);
 197                 ila = next;
 198         }
 199 }
 200
 201 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral);
 202
 203 static unsigned int
 204 ila_nf_input(void *priv,
 205              struct sk_buff *skb,
 206              const struct nf_hook_state *state)
 207 {
 208         ila_xlat_addr(skb, false);
 209         return NF_ACCEPT;
 210 }
 211
 212 static const struct nf_hook_ops ila_nf_hook_ops[] = {
 213         {
 214                 .hook = ila_nf_input,
 215                 .pf = NFPROTO_IPV6,
 216                 .hooknum = NF_INET_PRE_ROUTING,
 217                 .priority = -1,
 218         },
 219 };
 220
 221 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
 222 {
 223         struct ila_net *ilan = net_generic(net, ila_net_id);
 224         struct ila_map *ila, *head;
 225         spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
 226         int err = 0, order;
 227
 228         if (!ilan->hooks_registered) {
 229                 /* We defer registering net hooks in the namespace until the
 230                  * first mapping is added.
 231                  */
 232                 err = nf_register_net_hooks(net, ila_nf_hook_ops,
 233                                             ARRAY_SIZE(ila_nf_hook_ops));
 234                 if (err)
 235                         return err;
 236
 237                 ilan->hooks_registered = true;
 238         }
 239
 240         ila = kzalloc(sizeof(*ila), GFP_KERNEL);
 241         if (!ila)
 242                 return -ENOMEM;
 243
 244         ila_init_saved_csum(&xp->ip);
 245
 246         ila->xp = *xp;
 247
 248         order = ila_order(ila);
 249
 250         spin_lock(lock);
 251
 252         head = rhashtable_lookup_fast(&ilan->rhash_table,
 253                                       &xp->ip.locator_match,
 254                                       rht_params);
 255         if (!head) {
 256                 /* New entry for the rhash_table */
 257                 err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
 258                                                     &ila->node, rht_params);
 259         } else {
 260                 struct ila_map *tila = head, *prev = NULL;
 261
 262                 do {
 263                         if (!ila_cmp_params(tila, xp)) {
 264                                 err = -EEXIST;
 265                                 goto out;
 266                         }
 267
 268                         if (order > ila_order(tila))
 269                                 break;
 270
 271                         prev = tila;
 272                         tila = rcu_dereference_protected(tila->next,
 273                                 lockdep_is_held(lock));
 274                 } while (tila);
 275
 276                 if (prev) {
 277                         /* Insert in sub list of head */
 278                         RCU_INIT_POINTER(ila->next, tila);
 279                         rcu_assign_pointer(prev->next, ila);
 280                 } else {
 281                         /* Make this ila new head */
 282                         RCU_INIT_POINTER(ila->next, head);
 283                         err = rhashtable_replace_fast(&ilan->rhash_table,
 284                                                       &head->node,
 285                                                       &ila->node, rht_params);
 286                         if (err)
 287                                 goto out;
 288                 }
 289         }
 290
 291 out:
 292         spin_unlock(lock);
 293
 294         if (err)
 295                 kfree(ila);
 296
 297         return err;
 298 }
 299
 300 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
 301 {
 302         struct ila_net *ilan = net_generic(net, ila_net_id);
 303         struct ila_map *ila, *head, *prev;
 304         spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
 305         int err = -ENOENT;
 306
 307         spin_lock(lock);
 308
 309         head = rhashtable_lookup_fast(&ilan->rhash_table,
 310                                       &xp->ip.locator_match, rht_params);
 311         ila = head;
 312
 313         prev = NULL;
 314
 315         while (ila) {
 316                 if (ila_cmp_params(ila, xp)) {
 317                         prev = ila;
 318                         ila = rcu_dereference_protected(ila->next,
 319                                                         lockdep_is_held(lock));
 320                         continue;
 321                 }
 322
 323                 err = 0;
 324
 325                 if (prev) {
 326                         /* Not head, just delete from list */
 327                         rcu_assign_pointer(prev->next, ila->next);
 328                 } else {
 329                         /* It is the head. If there is something in the
 330                          * sublist we need to make a new head.
 331                          */
 332                         head = rcu_dereference_protected(ila->next,
 333                                                          lockdep_is_held(lock));
 334                         if (head) {
 335                                 /* Put first entry in the sublist into the
 336                                  * table
 337                                  */
 338                                 err = rhashtable_replace_fast(
 339                                         &ilan->rhash_table, &ila->node,
 340                                         &head->node, rht_params);
 341                                 if (err)
 342                                         goto out;
 343                         } else {
 344                                 /* Entry no longer used */
 345                                 err = rhashtable_remove_fast(&ilan->rhash_table,
 346                                                              &ila->node,
 347                                                              rht_params);
 348                         }
 349                 }
 350
 351                 ila_release(ila);
 352
 353                 break;
 354         }
 355
 356 out:
 357         spin_unlock(lock);
 358
 359         return err;
 360 }
 361
 362 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
 363 {
 364         struct net *net = genl_info_net(info);
 365         struct ila_xlat_params p;
 366         int err;
 367
 368         err = parse_nl_config(info, &p);
 369         if (err)
 370                 return err;
 371
 372         return ila_add_mapping(net, &p);
 373 }
 374
 375 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
 376 {
 377         struct net *net = genl_info_net(info);
 378         struct ila_xlat_params xp;
 379         int err;
 380
 381         err = parse_nl_config(info, &xp);
 382         if (err)
 383                 return err;
 384
 385         ila_del_mapping(net, &xp);
 386
 387         return 0;
 388 }
 389
 390 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
 391 {
 392         if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
 393                               (__force u64)ila->xp.ip.locator.v64,
 394                               ILA_ATTR_PAD) ||
 395             nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
 396                               (__force u64)ila->xp.ip.locator_match.v64,
 397                               ILA_ATTR_PAD) ||
 398             nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) ||
 399             nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode))
 400                 return -1;
 401
 402         return 0;
 403 }
 404
 405 static int ila_dump_info(struct ila_map *ila,
 406                          u32 portid, u32 seq, u32 flags,
 407                          struct sk_buff *skb, u8 cmd)
 408 {
 409         void *hdr;
 410
 411         hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
 412         if (!hdr)
 413                 return -ENOMEM;
 414
 415         if (ila_fill_info(ila, skb) < 0)
 416                 goto nla_put_failure;
 417
 418         genlmsg_end(skb, hdr);
 419         return 0;
 420
 421 nla_put_failure:
 422         genlmsg_cancel(skb, hdr);
 423         return -EMSGSIZE;
 424 }
 425
 426 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
 427 {
 428         struct net *net = genl_info_net(info);
 429         struct ila_net *ilan = net_generic(net, ila_net_id);
 430         struct sk_buff *msg;
 431         struct ila_xlat_params xp;
 432         struct ila_map *ila;
 433         int ret;
 434
 435         ret = parse_nl_config(info, &xp);
 436         if (ret)
 437                 return ret;
 438
 439         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 440         if (!msg)
 441                 return -ENOMEM;
 442
 443         rcu_read_lock();
 444
 445         ret = -ESRCH;
 446         ila = ila_lookup_by_params(&xp, ilan);
 447         if (ila) {
 448                 ret = ila_dump_info(ila,
 449                                     info->snd_portid,
 450                                     info->snd_seq, 0, msg,
 451                                     info->genlhdr->cmd);
 452         }
 453
 454         rcu_read_unlock();
 455
 456         if (ret < 0)
 457                 goto out_free;
 458
 459         return genlmsg_reply(msg, info);
 460
 461 out_free:
 462         nlmsg_free(msg);
 463         return ret;
 464 }
 465
 466 struct ila_dump_iter {
 467         struct rhashtable_iter rhiter;
 468 };
 469
 470 static int ila_nl_dump_start(struct netlink_callback *cb)
 471 {
 472         struct net *net = sock_net(cb->skb->sk);
 473         struct ila_net *ilan = net_generic(net, ila_net_id);
 474         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
 475
 476         if (!iter) {
 477                 iter = kmalloc(sizeof(*iter), GFP_KERNEL);
 478                 if (!iter)
 479                         return -ENOMEM;
 480
 481                 cb->args[0] = (long)iter;
 482         }
 483
 484         return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
 485                                     GFP_KERNEL);
 486 }
 487
 488 static int ila_nl_dump_done(struct netlink_callback *cb)
 489 {
 490         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
 491
 492         rhashtable_walk_exit(&iter->rhiter);
 493
 494         kfree(iter);
 495
 496         return 0;
 497 }
 498
 499 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
 500 {
 501         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
 502         struct rhashtable_iter *rhiter = &iter->rhiter;
 503         struct ila_map *ila;
 504         int ret;
 505
 506         ret = rhashtable_walk_start(rhiter);
 507         if (ret && ret != -EAGAIN)
 508                 goto done;
 509
 510         for (;;) {
 511                 ila = rhashtable_walk_next(rhiter);
 512
 513                 if (IS_ERR(ila)) {
 514                         if (PTR_ERR(ila) == -EAGAIN)
 515                                 continue;
 516                         ret = PTR_ERR(ila);
 517                         goto done;
 518                 } else if (!ila) {
 519                         break;
 520                 }
 521
 522                 while (ila) {
 523                         ret =  ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
 524                                              cb->nlh->nlmsg_seq, NLM_F_MULTI,
 525                                              skb, ILA_CMD_GET);
 526                         if (ret)
 527                                 goto done;
 528
 529                         ila = rcu_access_pointer(ila->next);
 530                 }
 531         }
 532
 533         ret = skb->len;
 534
 535 done:
 536         rhashtable_walk_stop(rhiter);
 537         return ret;
 538 }
 539
 540 static const struct genl_ops ila_nl_ops[] = {
 541         {
 542                 .cmd = ILA_CMD_ADD,
 543                 .doit = ila_nl_cmd_add_mapping,
 544                 .policy = ila_nl_policy,
 545                 .flags = GENL_ADMIN_PERM,
 546         },
 547         {
 548                 .cmd = ILA_CMD_DEL,
 549                 .doit = ila_nl_cmd_del_mapping,
 550                 .policy = ila_nl_policy,
 551                 .flags = GENL_ADMIN_PERM,
 552         },
 553         {
 554                 .cmd = ILA_CMD_GET,
 555                 .doit = ila_nl_cmd_get_mapping,
 556                 .start = ila_nl_dump_start,
 557                 .dumpit = ila_nl_dump,
 558                 .done = ila_nl_dump_done,
 559                 .policy = ila_nl_policy,
 560         },
 561 };
 562
 563 static struct genl_family ila_nl_family __ro_after_init = {
 564         .hdrsize        = 0,
 565         .name           = ILA_GENL_NAME,
 566         .version        = ILA_GENL_VERSION,
 567         .maxattr        = ILA_ATTR_MAX,
 568         .netnsok        = true,
 569         .parallel_ops   = true,
 570         .module         = THIS_MODULE,
 571         .ops            = ila_nl_ops,
 572         .n_ops          = ARRAY_SIZE(ila_nl_ops),
 573 };
 574
 575 #define ILA_HASH_TABLE_SIZE 1024
 576
 577 static __net_init int ila_init_net(struct net *net)
 578 {
 579         int err;
 580         struct ila_net *ilan = net_generic(net, ila_net_id);
 581
 582         err = alloc_ila_locks(ilan);
 583         if (err)
 584                 return err;
 585
 586         rhashtable_init(&ilan->rhash_table, &rht_params);
 587
 588         return 0;
 589 }
 590
 591 static __net_exit void ila_exit_net(struct net *net)
 592 {
 593         struct ila_net *ilan = net_generic(net, ila_net_id);
 594
 595         rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
 596
 597         kvfree(ilan->locks);
 598
 599         if (ilan->hooks_registered)
 600                 nf_unregister_net_hooks(net, ila_nf_hook_ops,
 601                                         ARRAY_SIZE(ila_nf_hook_ops));
 602 }
 603
 604 static struct pernet_operations ila_net_ops = {
 605         .init = ila_init_net,
 606         .exit = ila_exit_net,
 607         .id   = &ila_net_id,
 608         .size = sizeof(struct ila_net),
 609 };
 610
 611 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral)
 612 {
 613         struct ila_map *ila;
 614         struct ipv6hdr *ip6h = ipv6_hdr(skb);
 615         struct net *net = dev_net(skb->dev);
 616         struct ila_net *ilan = net_generic(net, ila_net_id);
 617         struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);
 618
 619         /* Assumes skb contains a valid IPv6 header that is pulled */
 620
 621         if (!ila_addr_is_ila(iaddr)) {
 622                 /* Type indicates this is not an ILA address */
 623                 return 0;
 624         }
 625
 626         rcu_read_lock();
 627
 628         ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
 629         if (ila)
 630                 ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral);
 631
 632         rcu_read_unlock();
 633
 634         return 0;
 635 }
 636
 637 int __init ila_xlat_init(void)
 638 {
 639         int ret;
 640
 641         ret = register_pernet_device(&ila_net_ops);
 642         if (ret)
 643                 goto exit;
 644
 645         ret = genl_register_family(&ila_nl_family);
 646         if (ret < 0)
 647                 goto unregister;
 648
 649         return 0;
 650
 651 unregister:
 652         unregister_pernet_device(&ila_net_ops);
 653 exit:
 654         return ret;
 655 }
 656
 657 void ila_xlat_fini(void)
 658 {
 659         genl_unregister_family(&ila_nl_family);
 660         unregister_pernet_device(&ila_net_ops);
 661 }