net/core/bpf_sk_storage.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019 Facebook  */
   3 #include <linux/rculist.h>
   4 #include <linux/list.h>
   5 #include <linux/hash.h>
   6 #include <linux/types.h>
   7 #include <linux/spinlock.h>
   8 #include <linux/bpf.h>
   9 #include <linux/btf_ids.h>
  10 #include <linux/bpf_local_storage.h>
  11 #include <net/bpf_sk_storage.h>
  12 #include <net/sock.h>
  13 #include <uapi/linux/sock_diag.h>
  14 #include <uapi/linux/btf.h>
  15
  16 DEFINE_BPF_STORAGE_CACHE(sk_cache);
  17
  18 static struct bpf_local_storage_data *
  19 sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
  20 {
  21         struct bpf_local_storage *sk_storage;
  22         struct bpf_local_storage_map *smap;
  23
  24         sk_storage = rcu_dereference(sk->sk_bpf_storage);
  25         if (!sk_storage)
  26                 return NULL;
  27
  28         smap = (struct bpf_local_storage_map *)map;
  29         return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit);
  30 }
  31
  32 static int sk_storage_delete(struct sock *sk, struct bpf_map *map)
  33 {
  34         struct bpf_local_storage_data *sdata;
  35
  36         sdata = sk_storage_lookup(sk, map, false);
  37         if (!sdata)
  38                 return -ENOENT;
  39
  40         bpf_selem_unlink(SELEM(sdata));
  41
  42         return 0;
  43 }
  44
  45 /* Called by __sk_destruct() & bpf_sk_storage_clone() */
  46 void bpf_sk_storage_free(struct sock *sk)
  47 {
  48         struct bpf_local_storage_elem *selem;
  49         struct bpf_local_storage *sk_storage;
  50         bool free_sk_storage = false;
  51         struct hlist_node *n;
  52
  53         rcu_read_lock();
  54         sk_storage = rcu_dereference(sk->sk_bpf_storage);
  55         if (!sk_storage) {
  56                 rcu_read_unlock();
  57                 return;
  58         }
  59
  60         /* Netiher the bpf_prog nor the bpf-map's syscall
  61          * could be modifying the sk_storage->list now.
  62          * Thus, no elem can be added-to or deleted-from the
  63          * sk_storage->list by the bpf_prog or by the bpf-map's syscall.
  64          *
  65          * It is racing with bpf_local_storage_map_free() alone
  66          * when unlinking elem from the sk_storage->list and
  67          * the map's bucket->list.
  68          */
  69         raw_spin_lock_bh(&sk_storage->lock);
  70         hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) {
  71                 /* Always unlink from map before unlinking from
  72                  * sk_storage.
  73                  */
  74                 bpf_selem_unlink_map(selem);
  75                 free_sk_storage = bpf_selem_unlink_storage_nolock(sk_storage,
  76                                                                   selem, true);
  77         }
  78         raw_spin_unlock_bh(&sk_storage->lock);
  79         rcu_read_unlock();
  80
  81         if (free_sk_storage)
  82                 kfree_rcu(sk_storage, rcu);
  83 }
  84
  85 static void sk_storage_map_free(struct bpf_map *map)
  86 {
  87         struct bpf_local_storage_map *smap;
  88
  89         smap = (struct bpf_local_storage_map *)map;
  90         bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx);
  91         bpf_local_storage_map_free(smap);
  92 }
  93
  94 static struct bpf_map *sk_storage_map_alloc(union bpf_attr *attr)
  95 {
  96         struct bpf_local_storage_map *smap;
  97
  98         smap = bpf_local_storage_map_alloc(attr);
  99         if (IS_ERR(smap))
 100                 return ERR_CAST(smap);
 101
 102         smap->cache_idx = bpf_local_storage_cache_idx_get(&sk_cache);
 103         return &smap->map;
 104 }
 105
 106 static int notsupp_get_next_key(struct bpf_map *map, void *key,
 107                                 void *next_key)
 108 {
 109         return -ENOTSUPP;
 110 }
 111
 112 static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
 113 {
 114         struct bpf_local_storage_data *sdata;
 115         struct socket *sock;
 116         int fd, err;
 117
 118         fd = *(int *)key;
 119         sock = sockfd_lookup(fd, &err);
 120         if (sock) {
 121                 sdata = sk_storage_lookup(sock->sk, map, true);
 122                 sockfd_put(sock);
 123                 return sdata ? sdata->data : NULL;
 124         }
 125
 126         return ERR_PTR(err);
 127 }
 128
 129 static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
 130                                          void *value, u64 map_flags)
 131 {
 132         struct bpf_local_storage_data *sdata;
 133         struct socket *sock;
 134         int fd, err;
 135
 136         fd = *(int *)key;
 137         sock = sockfd_lookup(fd, &err);
 138         if (sock) {
 139                 sdata = bpf_local_storage_update(
 140                         sock->sk, (struct bpf_local_storage_map *)map, value,
 141                         map_flags);
 142                 sockfd_put(sock);
 143                 return PTR_ERR_OR_ZERO(sdata);
 144         }
 145
 146         return err;
 147 }
 148
 149 static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
 150 {
 151         struct socket *sock;
 152         int fd, err;
 153
 154         fd = *(int *)key;
 155         sock = sockfd_lookup(fd, &err);
 156         if (sock) {
 157                 err = sk_storage_delete(sock->sk, map);
 158                 sockfd_put(sock);
 159                 return err;
 160         }
 161
 162         return err;
 163 }
 164
 165 static struct bpf_local_storage_elem *
 166 bpf_sk_storage_clone_elem(struct sock *newsk,
 167                           struct bpf_local_storage_map *smap,
 168                           struct bpf_local_storage_elem *selem)
 169 {
 170         struct bpf_local_storage_elem *copy_selem;
 171
 172         copy_selem = bpf_selem_alloc(smap, newsk, NULL, true);
 173         if (!copy_selem)
 174                 return NULL;
 175
 176         if (map_value_has_spin_lock(&smap->map))
 177                 copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
 178                                       SDATA(selem)->data, true);
 179         else
 180                 copy_map_value(&smap->map, SDATA(copy_selem)->data,
 181                                SDATA(selem)->data);
 182
 183         return copy_selem;
 184 }
 185
 186 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
 187 {
 188         struct bpf_local_storage *new_sk_storage = NULL;
 189         struct bpf_local_storage *sk_storage;
 190         struct bpf_local_storage_elem *selem;
 191         int ret = 0;
 192
 193         RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
 194
 195         rcu_read_lock();
 196         sk_storage = rcu_dereference(sk->sk_bpf_storage);
 197
 198         if (!sk_storage || hlist_empty(&sk_storage->list))
 199                 goto out;
 200
 201         hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
 202                 struct bpf_local_storage_elem *copy_selem;
 203                 struct bpf_local_storage_map *smap;
 204                 struct bpf_map *map;
 205
 206                 smap = rcu_dereference(SDATA(selem)->smap);
 207                 if (!(smap->map.map_flags & BPF_F_CLONE))
 208                         continue;
 209
 210                 /* Note that for lockless listeners adding new element
 211                  * here can race with cleanup in bpf_local_storage_map_free.
 212                  * Try to grab map refcnt to make sure that it's still
 213                  * alive and prevent concurrent removal.
 214                  */
 215                 map = bpf_map_inc_not_zero(&smap->map);
 216                 if (IS_ERR(map))
 217                         continue;
 218
 219                 copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
 220                 if (!copy_selem) {
 221                         ret = -ENOMEM;
 222                         bpf_map_put(map);
 223                         goto out;
 224                 }
 225
 226                 if (new_sk_storage) {
 227                         bpf_selem_link_map(smap, copy_selem);
 228                         bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
 229                 } else {
 230                         ret = bpf_local_storage_alloc(newsk, smap, copy_selem);
 231                         if (ret) {
 232                                 kfree(copy_selem);
 233                                 atomic_sub(smap->elem_size,
 234                                            &newsk->sk_omem_alloc);
 235                                 bpf_map_put(map);
 236                                 goto out;
 237                         }
 238
 239                         new_sk_storage =
 240                                 rcu_dereference(copy_selem->local_storage);
 241                 }
 242                 bpf_map_put(map);
 243         }
 244
 245 out:
 246         rcu_read_unlock();
 247
 248         /* In case of an error, don't free anything explicitly here, the
 249          * caller is responsible to call bpf_sk_storage_free.
 250          */
 251
 252         return ret;
 253 }
 254
 255 BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
 256            void *, value, u64, flags)
 257 {
 258         struct bpf_local_storage_data *sdata;
 259
 260         if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
 261                 return (unsigned long)NULL;
 262
 263         sdata = sk_storage_lookup(sk, map, true);
 264         if (sdata)
 265                 return (unsigned long)sdata->data;
 266
 267         if (flags == BPF_SK_STORAGE_GET_F_CREATE &&
 268             /* Cannot add new elem to a going away sk.
 269              * Otherwise, the new elem may become a leak
 270              * (and also other memory issues during map
 271              *  destruction).
 272              */
 273             refcount_inc_not_zero(&sk->sk_refcnt)) {
 274                 sdata = bpf_local_storage_update(
 275                         sk, (struct bpf_local_storage_map *)map, value,
 276                         BPF_NOEXIST);
 277                 /* sk must be a fullsock (guaranteed by verifier),
 278                  * so sock_gen_put() is unnecessary.
 279                  */
 280                 sock_put(sk);
 281                 return IS_ERR(sdata) ?
 282                         (unsigned long)NULL : (unsigned long)sdata->data;
 283         }
 284
 285         return (unsigned long)NULL;
 286 }
 287
 288 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
 289 {
 290         if (!sk || !sk_fullsock(sk))
 291                 return -EINVAL;
 292
 293         if (refcount_inc_not_zero(&sk->sk_refcnt)) {
 294                 int err;
 295
 296                 err = sk_storage_delete(sk, map);
 297                 sock_put(sk);
 298                 return err;
 299         }
 300
 301         return -ENOENT;
 302 }
 303
 304 static int sk_storage_charge(struct bpf_local_storage_map *smap,
 305                              void *owner, u32 size)
 306 {
 307         int optmem_max = READ_ONCE(sysctl_optmem_max);
 308         struct sock *sk = (struct sock *)owner;
 309
 310         /* same check as in sock_kmalloc() */
 311         if (size <= optmem_max &&
 312             atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
 313                 atomic_add(size, &sk->sk_omem_alloc);
 314                 return 0;
 315         }
 316
 317         return -ENOMEM;
 318 }
 319
 320 static void sk_storage_uncharge(struct bpf_local_storage_map *smap,
 321                                 void *owner, u32 size)
 322 {
 323         struct sock *sk = owner;
 324
 325         atomic_sub(size, &sk->sk_omem_alloc);
 326 }
 327
 328 static struct bpf_local_storage __rcu **
 329 sk_storage_ptr(void *owner)
 330 {
 331         struct sock *sk = owner;
 332
 333         return &sk->sk_bpf_storage;
 334 }
 335
 336 static int sk_storage_map_btf_id;
 337 const struct bpf_map_ops sk_storage_map_ops = {
 338         .map_meta_equal = bpf_map_meta_equal,
 339         .map_alloc_check = bpf_local_storage_map_alloc_check,
 340         .map_alloc = sk_storage_map_alloc,
 341         .map_free = sk_storage_map_free,
 342         .map_get_next_key = notsupp_get_next_key,
 343         .map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
 344         .map_update_elem = bpf_fd_sk_storage_update_elem,
 345         .map_delete_elem = bpf_fd_sk_storage_delete_elem,
 346         .map_check_btf = bpf_local_storage_map_check_btf,
 347         .map_btf_name = "bpf_local_storage_map",
 348         .map_btf_id = &sk_storage_map_btf_id,
 349         .map_local_storage_charge = sk_storage_charge,
 350         .map_local_storage_uncharge = sk_storage_uncharge,
 351         .map_owner_storage_ptr = sk_storage_ptr,
 352 };
 353
 354 const struct bpf_func_proto bpf_sk_storage_get_proto = {
 355         .func           = bpf_sk_storage_get,
 356         .gpl_only       = false,
 357         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 358         .arg1_type      = ARG_CONST_MAP_PTR,
 359         .arg2_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 360         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 361         .arg4_type      = ARG_ANYTHING,
 362 };
 363
 364 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = {
 365         .func           = bpf_sk_storage_get,
 366         .gpl_only       = false,
 367         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 368         .arg1_type      = ARG_CONST_MAP_PTR,
 369         .arg2_type      = ARG_PTR_TO_CTX, /* context is 'struct sock' */
 370         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 371         .arg4_type      = ARG_ANYTHING,
 372 };
 373
 374 const struct bpf_func_proto bpf_sk_storage_delete_proto = {
 375         .func           = bpf_sk_storage_delete,
 376         .gpl_only       = false,
 377         .ret_type       = RET_INTEGER,
 378         .arg1_type      = ARG_CONST_MAP_PTR,
 379         .arg2_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 380 };
 381
 382 struct bpf_sk_storage_diag {
 383         u32 nr_maps;
 384         struct bpf_map *maps[];
 385 };
 386
 387 /* The reply will be like:
 388  * INET_DIAG_BPF_SK_STORAGES (nla_nest)
 389  *      SK_DIAG_BPF_STORAGE (nla_nest)
 390  *              SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
 391  *              SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
 392  *      SK_DIAG_BPF_STORAGE (nla_nest)
 393  *              SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
 394  *              SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
 395  *      ....
 396  */
 397 static int nla_value_size(u32 value_size)
 398 {
 399         /* SK_DIAG_BPF_STORAGE (nla_nest)
 400          *      SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
 401          *      SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
 402          */
 403         return nla_total_size(0) + nla_total_size(sizeof(u32)) +
 404                 nla_total_size_64bit(value_size);
 405 }
 406
 407 void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag)
 408 {
 409         u32 i;
 410
 411         if (!diag)
 412                 return;
 413
 414         for (i = 0; i < diag->nr_maps; i++)
 415                 bpf_map_put(diag->maps[i]);
 416
 417         kfree(diag);
 418 }
 419 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free);
 420
 421 static bool diag_check_dup(const struct bpf_sk_storage_diag *diag,
 422                            const struct bpf_map *map)
 423 {
 424         u32 i;
 425
 426         for (i = 0; i < diag->nr_maps; i++) {
 427                 if (diag->maps[i] == map)
 428                         return true;
 429         }
 430
 431         return false;
 432 }
 433
 434 struct bpf_sk_storage_diag *
 435 bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs)
 436 {
 437         struct bpf_sk_storage_diag *diag;
 438         struct nlattr *nla;
 439         u32 nr_maps = 0;
 440         int rem, err;
 441
 442         /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as
 443          * the map_alloc_check() side also does.
 444          */
 445         if (!bpf_capable())
 446                 return ERR_PTR(-EPERM);
 447
 448         nla_for_each_nested(nla, nla_stgs, rem) {
 449                 if (nla_type(nla) == SK_DIAG_BPF_STORAGE_REQ_MAP_FD) {
 450                         if (nla_len(nla) != sizeof(u32))
 451                                 return ERR_PTR(-EINVAL);
 452                         nr_maps++;
 453                 }
 454         }
 455
 456         diag = kzalloc(sizeof(*diag) + sizeof(diag->maps[0]) * nr_maps,
 457                        GFP_KERNEL);
 458         if (!diag)
 459                 return ERR_PTR(-ENOMEM);
 460
 461         nla_for_each_nested(nla, nla_stgs, rem) {
 462                 struct bpf_map *map;
 463                 int map_fd;
 464
 465                 if (nla_type(nla) != SK_DIAG_BPF_STORAGE_REQ_MAP_FD)
 466                         continue;
 467
 468                 map_fd = nla_get_u32(nla);
 469                 map = bpf_map_get(map_fd);
 470                 if (IS_ERR(map)) {
 471                         err = PTR_ERR(map);
 472                         goto err_free;
 473                 }
 474                 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) {
 475                         bpf_map_put(map);
 476                         err = -EINVAL;
 477                         goto err_free;
 478                 }
 479                 if (diag_check_dup(diag, map)) {
 480                         bpf_map_put(map);
 481                         err = -EEXIST;
 482                         goto err_free;
 483                 }
 484                 diag->maps[diag->nr_maps++] = map;
 485         }
 486
 487         return diag;
 488
 489 err_free:
 490         bpf_sk_storage_diag_free(diag);
 491         return ERR_PTR(err);
 492 }
 493 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
 494
 495 static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
 496 {
 497         struct nlattr *nla_stg, *nla_value;
 498         struct bpf_local_storage_map *smap;
 499
 500         /* It cannot exceed max nlattr's payload */
 501         BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
 502
 503         nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE);
 504         if (!nla_stg)
 505                 return -EMSGSIZE;
 506
 507         smap = rcu_dereference(sdata->smap);
 508         if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
 509                 goto errout;
 510
 511         nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE,
 512                                       smap->map.value_size,
 513                                       SK_DIAG_BPF_STORAGE_PAD);
 514         if (!nla_value)
 515                 goto errout;
 516
 517         if (map_value_has_spin_lock(&smap->map))
 518                 copy_map_value_locked(&smap->map, nla_data(nla_value),
 519                                       sdata->data, true);
 520         else
 521                 copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
 522
 523         nla_nest_end(skb, nla_stg);
 524         return 0;
 525
 526 errout:
 527         nla_nest_cancel(skb, nla_stg);
 528         return -EMSGSIZE;
 529 }
 530
 531 static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
 532                                        int stg_array_type,
 533                                        unsigned int *res_diag_size)
 534 {
 535         /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
 536         unsigned int diag_size = nla_total_size(0);
 537         struct bpf_local_storage *sk_storage;
 538         struct bpf_local_storage_elem *selem;
 539         struct bpf_local_storage_map *smap;
 540         struct nlattr *nla_stgs;
 541         unsigned int saved_len;
 542         int err = 0;
 543
 544         rcu_read_lock();
 545
 546         sk_storage = rcu_dereference(sk->sk_bpf_storage);
 547         if (!sk_storage || hlist_empty(&sk_storage->list)) {
 548                 rcu_read_unlock();
 549                 return 0;
 550         }
 551
 552         nla_stgs = nla_nest_start(skb, stg_array_type);
 553         if (!nla_stgs)
 554                 /* Continue to learn diag_size */
 555                 err = -EMSGSIZE;
 556
 557         saved_len = skb->len;
 558         hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
 559                 smap = rcu_dereference(SDATA(selem)->smap);
 560                 diag_size += nla_value_size(smap->map.value_size);
 561
 562                 if (nla_stgs && diag_get(SDATA(selem), skb))
 563                         /* Continue to learn diag_size */
 564                         err = -EMSGSIZE;
 565         }
 566
 567         rcu_read_unlock();
 568
 569         if (nla_stgs) {
 570                 if (saved_len == skb->len)
 571                         nla_nest_cancel(skb, nla_stgs);
 572                 else
 573                         nla_nest_end(skb, nla_stgs);
 574         }
 575
 576         if (diag_size == nla_total_size(0)) {
 577                 *res_diag_size = 0;
 578                 return 0;
 579         }
 580
 581         *res_diag_size = diag_size;
 582         return err;
 583 }
 584
 585 int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
 586                             struct sock *sk, struct sk_buff *skb,
 587                             int stg_array_type,
 588                             unsigned int *res_diag_size)
 589 {
 590         /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
 591         unsigned int diag_size = nla_total_size(0);
 592         struct bpf_local_storage *sk_storage;
 593         struct bpf_local_storage_data *sdata;
 594         struct nlattr *nla_stgs;
 595         unsigned int saved_len;
 596         int err = 0;
 597         u32 i;
 598
 599         *res_diag_size = 0;
 600
 601         /* No map has been specified.  Dump all. */
 602         if (!diag->nr_maps)
 603                 return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type,
 604                                                    res_diag_size);
 605
 606         rcu_read_lock();
 607         sk_storage = rcu_dereference(sk->sk_bpf_storage);
 608         if (!sk_storage || hlist_empty(&sk_storage->list)) {
 609                 rcu_read_unlock();
 610                 return 0;
 611         }
 612
 613         nla_stgs = nla_nest_start(skb, stg_array_type);
 614         if (!nla_stgs)
 615                 /* Continue to learn diag_size */
 616                 err = -EMSGSIZE;
 617
 618         saved_len = skb->len;
 619         for (i = 0; i < diag->nr_maps; i++) {
 620                 sdata = bpf_local_storage_lookup(sk_storage,
 621                                 (struct bpf_local_storage_map *)diag->maps[i],
 622                                 false);
 623
 624                 if (!sdata)
 625                         continue;
 626
 627                 diag_size += nla_value_size(diag->maps[i]->value_size);
 628
 629                 if (nla_stgs && diag_get(sdata, skb))
 630                         /* Continue to learn diag_size */
 631                         err = -EMSGSIZE;
 632         }
 633         rcu_read_unlock();
 634
 635         if (nla_stgs) {
 636                 if (saved_len == skb->len)
 637                         nla_nest_cancel(skb, nla_stgs);
 638                 else
 639                         nla_nest_end(skb, nla_stgs);
 640         }
 641
 642         if (diag_size == nla_total_size(0)) {
 643                 *res_diag_size = 0;
 644                 return 0;
 645         }
 646
 647         *res_diag_size = diag_size;
 648         return err;
 649 }
 650 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put);
 651
 652 struct bpf_iter_seq_sk_storage_map_info {
 653         struct bpf_map *map;
 654         unsigned int bucket_id;
 655         unsigned skip_elems;
 656 };
 657
 658 static struct bpf_local_storage_elem *
 659 bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info,
 660                                  struct bpf_local_storage_elem *prev_selem)
 661         __acquires(RCU) __releases(RCU)
 662 {
 663         struct bpf_local_storage *sk_storage;
 664         struct bpf_local_storage_elem *selem;
 665         u32 skip_elems = info->skip_elems;
 666         struct bpf_local_storage_map *smap;
 667         u32 bucket_id = info->bucket_id;
 668         u32 i, count, n_buckets;
 669         struct bpf_local_storage_map_bucket *b;
 670
 671         smap = (struct bpf_local_storage_map *)info->map;
 672         n_buckets = 1U << smap->bucket_log;
 673         if (bucket_id >= n_buckets)
 674                 return NULL;
 675
 676         /* try to find next selem in the same bucket */
 677         selem = prev_selem;
 678         count = 0;
 679         while (selem) {
 680                 selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)),
 681                                          struct bpf_local_storage_elem, map_node);
 682                 if (!selem) {
 683                         /* not found, unlock and go to the next bucket */
 684                         b = &smap->buckets[bucket_id++];
 685                         rcu_read_unlock();
 686                         skip_elems = 0;
 687                         break;
 688                 }
 689                 sk_storage = rcu_dereference(selem->local_storage);
 690                 if (sk_storage) {
 691                         info->skip_elems = skip_elems + count;
 692                         return selem;
 693                 }
 694                 count++;
 695         }
 696
 697         for (i = bucket_id; i < (1U << smap->bucket_log); i++) {
 698                 b = &smap->buckets[i];
 699                 rcu_read_lock();
 700                 count = 0;
 701                 hlist_for_each_entry_rcu(selem, &b->list, map_node) {
 702                         sk_storage = rcu_dereference(selem->local_storage);
 703                         if (sk_storage && count >= skip_elems) {
 704                                 info->bucket_id = i;
 705                                 info->skip_elems = count;
 706                                 return selem;
 707                         }
 708                         count++;
 709                 }
 710                 rcu_read_unlock();
 711                 skip_elems = 0;
 712         }
 713
 714         info->bucket_id = i;
 715         info->skip_elems = 0;
 716         return NULL;
 717 }
 718
 719 static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos)
 720 {
 721         struct bpf_local_storage_elem *selem;
 722
 723         selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL);
 724         if (!selem)
 725                 return NULL;
 726
 727         if (*pos == 0)
 728                 ++*pos;
 729         return selem;
 730 }
 731
 732 static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v,
 733                                          loff_t *pos)
 734 {
 735         struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
 736
 737         ++*pos;
 738         ++info->skip_elems;
 739         return bpf_sk_storage_map_seq_find_next(seq->private, v);
 740 }
 741
 742 struct bpf_iter__bpf_sk_storage_map {
 743         __bpf_md_ptr(struct bpf_iter_meta *, meta);
 744         __bpf_md_ptr(struct bpf_map *, map);
 745         __bpf_md_ptr(struct sock *, sk);
 746         __bpf_md_ptr(void *, value);
 747 };
 748
 749 DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta,
 750                      struct bpf_map *map, struct sock *sk,
 751                      void *value)
 752
 753 static int __bpf_sk_storage_map_seq_show(struct seq_file *seq,
 754                                          struct bpf_local_storage_elem *selem)
 755 {
 756         struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
 757         struct bpf_iter__bpf_sk_storage_map ctx = {};
 758         struct bpf_local_storage *sk_storage;
 759         struct bpf_iter_meta meta;
 760         struct bpf_prog *prog;
 761         int ret = 0;
 762
 763         meta.seq = seq;
 764         prog = bpf_iter_get_info(&meta, selem == NULL);
 765         if (prog) {
 766                 ctx.meta = &meta;
 767                 ctx.map = info->map;
 768                 if (selem) {
 769                         sk_storage = rcu_dereference(selem->local_storage);
 770                         ctx.sk = sk_storage->owner;
 771                         ctx.value = SDATA(selem)->data;
 772                 }
 773                 ret = bpf_iter_run_prog(prog, &ctx);
 774         }
 775
 776         return ret;
 777 }
 778
 779 static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v)
 780 {
 781         return __bpf_sk_storage_map_seq_show(seq, v);
 782 }
 783
 784 static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v)
 785         __releases(RCU)
 786 {
 787         if (!v)
 788                 (void)__bpf_sk_storage_map_seq_show(seq, v);
 789         else
 790                 rcu_read_unlock();
 791 }
 792
 793 static int bpf_iter_init_sk_storage_map(void *priv_data,
 794                                         struct bpf_iter_aux_info *aux)
 795 {
 796         struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
 797
 798         bpf_map_inc_with_uref(aux->map);
 799         seq_info->map = aux->map;
 800         return 0;
 801 }
 802
 803 static void bpf_iter_fini_sk_storage_map(void *priv_data)
 804 {
 805         struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
 806
 807         bpf_map_put_with_uref(seq_info->map);
 808 }
 809
 810 static int bpf_iter_attach_map(struct bpf_prog *prog,
 811                                union bpf_iter_link_info *linfo,
 812                                struct bpf_iter_aux_info *aux)
 813 {
 814         struct bpf_map *map;
 815         int err = -EINVAL;
 816
 817         if (!linfo->map.map_fd)
 818                 return -EBADF;
 819
 820         map = bpf_map_get_with_uref(linfo->map.map_fd);
 821         if (IS_ERR(map))
 822                 return PTR_ERR(map);
 823
 824         if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
 825                 goto put_map;
 826
 827         if (prog->aux->max_rdwr_access > map->value_size) {
 828                 err = -EACCES;
 829                 goto put_map;
 830         }
 831
 832         aux->map = map;
 833         return 0;
 834
 835 put_map:
 836         bpf_map_put_with_uref(map);
 837         return err;
 838 }
 839
 840 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux)
 841 {
 842         bpf_map_put_with_uref(aux->map);
 843 }
 844
 845 static const struct seq_operations bpf_sk_storage_map_seq_ops = {
 846         .start  = bpf_sk_storage_map_seq_start,
 847         .next   = bpf_sk_storage_map_seq_next,
 848         .stop   = bpf_sk_storage_map_seq_stop,
 849         .show   = bpf_sk_storage_map_seq_show,
 850 };
 851
 852 static const struct bpf_iter_seq_info iter_seq_info = {
 853         .seq_ops                = &bpf_sk_storage_map_seq_ops,
 854         .init_seq_private       = bpf_iter_init_sk_storage_map,
 855         .fini_seq_private       = bpf_iter_fini_sk_storage_map,
 856         .seq_priv_size          = sizeof(struct bpf_iter_seq_sk_storage_map_info),
 857 };
 858
 859 static struct bpf_iter_reg bpf_sk_storage_map_reg_info = {
 860         .target                 = "bpf_sk_storage_map",
 861         .attach_target          = bpf_iter_attach_map,
 862         .detach_target          = bpf_iter_detach_map,
 863         .show_fdinfo            = bpf_iter_map_show_fdinfo,
 864         .fill_link_info         = bpf_iter_map_fill_link_info,
 865         .ctx_arg_info_size      = 2,
 866         .ctx_arg_info           = {
 867                 { offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
 868                   PTR_TO_BTF_ID_OR_NULL },
 869                 { offsetof(struct bpf_iter__bpf_sk_storage_map, value),
 870                   PTR_TO_RDWR_BUF_OR_NULL },
 871         },
 872         .seq_info               = &iter_seq_info,
 873 };
 874
 875 static int __init bpf_sk_storage_map_iter_init(void)
 876 {
 877         bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id =
 878                 btf_sock_ids[BTF_SOCK_TYPE_SOCK];
 879         return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info);
 880 }
 881 late_initcall(bpf_sk_storage_map_iter_init);