net/core/bpf_sk_storage.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019 Facebook  */
   3 #include <linux/rculist.h>
   4 #include <linux/list.h>
   5 #include <linux/hash.h>
   6 #include <linux/types.h>
   7 #include <linux/spinlock.h>
   8 #include <linux/bpf.h>
   9 #include <linux/btf.h>
  10 #include <linux/btf_ids.h>
  11 #include <linux/bpf_local_storage.h>
  12 #include <net/bpf_sk_storage.h>
  13 #include <net/sock.h>
  14 #include <uapi/linux/sock_diag.h>
  15 #include <uapi/linux/btf.h>
  16 #include <linux/rcupdate_trace.h>
  17
  18 DEFINE_BPF_STORAGE_CACHE(sk_cache);
  19
  20 static struct bpf_local_storage_data *
  21 bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
  22 {
  23         struct bpf_local_storage *sk_storage;
  24         struct bpf_local_storage_map *smap;
  25
  26         sk_storage =
  27                 rcu_dereference_check(sk->sk_bpf_storage, bpf_rcu_lock_held());
  28         if (!sk_storage)
  29                 return NULL;
  30
  31         smap = (struct bpf_local_storage_map *)map;
  32         return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit);
  33 }
  34
  35 static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
  36 {
  37         struct bpf_local_storage_data *sdata;
  38
  39         sdata = bpf_sk_storage_lookup(sk, map, false);
  40         if (!sdata)
  41                 return -ENOENT;
  42
  43         bpf_selem_unlink(SELEM(sdata), false);
  44
  45         return 0;
  46 }
  47
  48 /* Called by __sk_destruct() & bpf_sk_storage_clone() */
  49 void bpf_sk_storage_free(struct sock *sk)
  50 {
  51         struct bpf_local_storage *sk_storage;
  52
  53         rcu_read_lock();
  54         sk_storage = rcu_dereference(sk->sk_bpf_storage);
  55         if (!sk_storage) {
  56                 rcu_read_unlock();
  57                 return;
  58         }
  59
  60         bpf_local_storage_destroy(sk_storage);
  61         rcu_read_unlock();
  62 }
  63
  64 static void bpf_sk_storage_map_free(struct bpf_map *map)
  65 {
  66         bpf_local_storage_map_free(map, &sk_cache, NULL);
  67 }
  68
  69 static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
  70 {
  71         return bpf_local_storage_map_alloc(attr, &sk_cache, false);
  72 }
  73
  74 static int notsupp_get_next_key(struct bpf_map *map, void *key,
  75                                 void *next_key)
  76 {
  77         return -ENOTSUPP;
  78 }
  79
  80 static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
  81 {
  82         struct bpf_local_storage_data *sdata;
  83         struct socket *sock;
  84         int fd, err;
  85
  86         fd = *(int *)key;
  87         sock = sockfd_lookup(fd, &err);
  88         if (sock) {
  89                 sdata = bpf_sk_storage_lookup(sock->sk, map, true);
  90                 sockfd_put(sock);
  91                 return sdata ? sdata->data : NULL;
  92         }
  93
  94         return ERR_PTR(err);
  95 }
  96
  97 static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
  98                                           void *value, u64 map_flags)
  99 {
 100         struct bpf_local_storage_data *sdata;
 101         struct socket *sock;
 102         int fd, err;
 103
 104         fd = *(int *)key;
 105         sock = sockfd_lookup(fd, &err);
 106         if (sock) {
 107                 sdata = bpf_local_storage_update(
 108                         sock->sk, (struct bpf_local_storage_map *)map, value,
 109                         map_flags, GFP_ATOMIC);
 110                 sockfd_put(sock);
 111                 return PTR_ERR_OR_ZERO(sdata);
 112         }
 113
 114         return err;
 115 }
 116
 117 static long bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
 118 {
 119         struct socket *sock;
 120         int fd, err;
 121
 122         fd = *(int *)key;
 123         sock = sockfd_lookup(fd, &err);
 124         if (sock) {
 125                 err = bpf_sk_storage_del(sock->sk, map);
 126                 sockfd_put(sock);
 127                 return err;
 128         }
 129
 130         return err;
 131 }
 132
 133 static struct bpf_local_storage_elem *
 134 bpf_sk_storage_clone_elem(struct sock *newsk,
 135                           struct bpf_local_storage_map *smap,
 136                           struct bpf_local_storage_elem *selem)
 137 {
 138         struct bpf_local_storage_elem *copy_selem;
 139
 140         copy_selem = bpf_selem_alloc(smap, newsk, NULL, true, GFP_ATOMIC);
 141         if (!copy_selem)
 142                 return NULL;
 143
 144         if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK))
 145                 copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
 146                                       SDATA(selem)->data, true);
 147         else
 148                 copy_map_value(&smap->map, SDATA(copy_selem)->data,
 149                                SDATA(selem)->data);
 150
 151         return copy_selem;
 152 }
 153
 154 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
 155 {
 156         struct bpf_local_storage *new_sk_storage = NULL;
 157         struct bpf_local_storage *sk_storage;
 158         struct bpf_local_storage_elem *selem;
 159         int ret = 0;
 160
 161         RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
 162
 163         rcu_read_lock();
 164         sk_storage = rcu_dereference(sk->sk_bpf_storage);
 165
 166         if (!sk_storage || hlist_empty(&sk_storage->list))
 167                 goto out;
 168
 169         hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
 170                 struct bpf_local_storage_elem *copy_selem;
 171                 struct bpf_local_storage_map *smap;
 172                 struct bpf_map *map;
 173
 174                 smap = rcu_dereference(SDATA(selem)->smap);
 175                 if (!(smap->map.map_flags & BPF_F_CLONE))
 176                         continue;
 177
 178                 /* Note that for lockless listeners adding new element
 179                  * here can race with cleanup in bpf_local_storage_map_free.
 180                  * Try to grab map refcnt to make sure that it's still
 181                  * alive and prevent concurrent removal.
 182                  */
 183                 map = bpf_map_inc_not_zero(&smap->map);
 184                 if (IS_ERR(map))
 185                         continue;
 186
 187                 copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
 188                 if (!copy_selem) {
 189                         ret = -ENOMEM;
 190                         bpf_map_put(map);
 191                         goto out;
 192                 }
 193
 194                 if (new_sk_storage) {
 195                         bpf_selem_link_map(smap, copy_selem);
 196                         bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
 197                 } else {
 198                         ret = bpf_local_storage_alloc(newsk, smap, copy_selem, GFP_ATOMIC);
 199                         if (ret) {
 200                                 bpf_selem_free(copy_selem, smap, true);
 201                                 atomic_sub(smap->elem_size,
 202                                            &newsk->sk_omem_alloc);
 203                                 bpf_map_put(map);
 204                                 goto out;
 205                         }
 206
 207                         new_sk_storage =
 208                                 rcu_dereference(copy_selem->local_storage);
 209                 }
 210                 bpf_map_put(map);
 211         }
 212
 213 out:
 214         rcu_read_unlock();
 215
 216         /* In case of an error, don't free anything explicitly here, the
 217          * caller is responsible to call bpf_sk_storage_free.
 218          */
 219
 220         return ret;
 221 }
 222
 223 /* *gfp_flags* is a hidden argument provided by the verifier */
 224 BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
 225            void *, value, u64, flags, gfp_t, gfp_flags)
 226 {
 227         struct bpf_local_storage_data *sdata;
 228
 229         WARN_ON_ONCE(!bpf_rcu_lock_held());
 230         if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
 231                 return (unsigned long)NULL;
 232
 233         sdata = bpf_sk_storage_lookup(sk, map, true);
 234         if (sdata)
 235                 return (unsigned long)sdata->data;
 236
 237         if (flags == BPF_SK_STORAGE_GET_F_CREATE &&
 238             /* Cannot add new elem to a going away sk.
 239              * Otherwise, the new elem may become a leak
 240              * (and also other memory issues during map
 241              *  destruction).
 242              */
 243             refcount_inc_not_zero(&sk->sk_refcnt)) {
 244                 sdata = bpf_local_storage_update(
 245                         sk, (struct bpf_local_storage_map *)map, value,
 246                         BPF_NOEXIST, gfp_flags);
 247                 /* sk must be a fullsock (guaranteed by verifier),
 248                  * so sock_gen_put() is unnecessary.
 249                  */
 250                 sock_put(sk);
 251                 return IS_ERR(sdata) ?
 252                         (unsigned long)NULL : (unsigned long)sdata->data;
 253         }
 254
 255         return (unsigned long)NULL;
 256 }
 257
 258 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
 259 {
 260         WARN_ON_ONCE(!bpf_rcu_lock_held());
 261         if (!sk || !sk_fullsock(sk))
 262                 return -EINVAL;
 263
 264         if (refcount_inc_not_zero(&sk->sk_refcnt)) {
 265                 int err;
 266
 267                 err = bpf_sk_storage_del(sk, map);
 268                 sock_put(sk);
 269                 return err;
 270         }
 271
 272         return -ENOENT;
 273 }
 274
 275 static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap,
 276                                  void *owner, u32 size)
 277 {
 278         struct sock *sk = (struct sock *)owner;
 279         int optmem_max;
 280
 281         optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
 282         /* same check as in sock_kmalloc() */
 283         if (size <= optmem_max &&
 284             atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
 285                 atomic_add(size, &sk->sk_omem_alloc);
 286                 return 0;
 287         }
 288
 289         return -ENOMEM;
 290 }
 291
 292 static void bpf_sk_storage_uncharge(struct bpf_local_storage_map *smap,
 293                                     void *owner, u32 size)
 294 {
 295         struct sock *sk = owner;
 296
 297         atomic_sub(size, &sk->sk_omem_alloc);
 298 }
 299
 300 static struct bpf_local_storage __rcu **
 301 bpf_sk_storage_ptr(void *owner)
 302 {
 303         struct sock *sk = owner;
 304
 305         return &sk->sk_bpf_storage;
 306 }
 307
 308 const struct bpf_map_ops sk_storage_map_ops = {
 309         .map_meta_equal = bpf_map_meta_equal,
 310         .map_alloc_check = bpf_local_storage_map_alloc_check,
 311         .map_alloc = bpf_sk_storage_map_alloc,
 312         .map_free = bpf_sk_storage_map_free,
 313         .map_get_next_key = notsupp_get_next_key,
 314         .map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
 315         .map_update_elem = bpf_fd_sk_storage_update_elem,
 316         .map_delete_elem = bpf_fd_sk_storage_delete_elem,
 317         .map_check_btf = bpf_local_storage_map_check_btf,
 318         .map_btf_id = &bpf_local_storage_map_btf_id[0],
 319         .map_local_storage_charge = bpf_sk_storage_charge,
 320         .map_local_storage_uncharge = bpf_sk_storage_uncharge,
 321         .map_owner_storage_ptr = bpf_sk_storage_ptr,
 322         .map_mem_usage = bpf_local_storage_map_mem_usage,
 323 };
 324
 325 const struct bpf_func_proto bpf_sk_storage_get_proto = {
 326         .func           = bpf_sk_storage_get,
 327         .gpl_only       = false,
 328         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 329         .arg1_type      = ARG_CONST_MAP_PTR,
 330         .arg2_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 331         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 332         .arg4_type      = ARG_ANYTHING,
 333 };
 334
 335 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = {
 336         .func           = bpf_sk_storage_get,
 337         .gpl_only       = false,
 338         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 339         .arg1_type      = ARG_CONST_MAP_PTR,
 340         .arg2_type      = ARG_PTR_TO_CTX, /* context is 'struct sock' */
 341         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 342         .arg4_type      = ARG_ANYTHING,
 343 };
 344
 345 const struct bpf_func_proto bpf_sk_storage_delete_proto = {
 346         .func           = bpf_sk_storage_delete,
 347         .gpl_only       = false,
 348         .ret_type       = RET_INTEGER,
 349         .arg1_type      = ARG_CONST_MAP_PTR,
 350         .arg2_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 351 };
 352
 353 static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)
 354 {
 355         const struct btf *btf_vmlinux;
 356         const struct btf_type *t;
 357         const char *tname;
 358         u32 btf_id;
 359
 360         if (prog->aux->dst_prog)
 361                 return false;
 362
 363         /* Ensure the tracing program is not tracing
 364          * any bpf_sk_storage*() function and also
 365          * use the bpf_sk_storage_(get|delete) helper.
 366          */
 367         switch (prog->expected_attach_type) {
 368         case BPF_TRACE_ITER:
 369         case BPF_TRACE_RAW_TP:
 370                 /* bpf_sk_storage has no trace point */
 371                 return true;
 372         case BPF_TRACE_FENTRY:
 373         case BPF_TRACE_FEXIT:
 374                 btf_vmlinux = bpf_get_btf_vmlinux();
 375                 if (IS_ERR_OR_NULL(btf_vmlinux))
 376                         return false;
 377                 btf_id = prog->aux->attach_btf_id;
 378                 t = btf_type_by_id(btf_vmlinux, btf_id);
 379                 tname = btf_name_by_offset(btf_vmlinux, t->name_off);
 380                 return !!strncmp(tname, "bpf_sk_storage",
 381                                  strlen("bpf_sk_storage"));
 382         default:
 383                 return false;
 384         }
 385
 386         return false;
 387 }
 388
 389 /* *gfp_flags* is a hidden argument provided by the verifier */
 390 BPF_CALL_5(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
 391            void *, value, u64, flags, gfp_t, gfp_flags)
 392 {
 393         WARN_ON_ONCE(!bpf_rcu_lock_held());
 394         if (in_hardirq() || in_nmi())
 395                 return (unsigned long)NULL;
 396
 397         return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags,
 398                                                      gfp_flags);
 399 }
 400
 401 BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,
 402            struct sock *, sk)
 403 {
 404         WARN_ON_ONCE(!bpf_rcu_lock_held());
 405         if (in_hardirq() || in_nmi())
 406                 return -EPERM;
 407
 408         return ____bpf_sk_storage_delete(map, sk);
 409 }
 410
 411 const struct bpf_func_proto bpf_sk_storage_get_tracing_proto = {
 412         .func           = bpf_sk_storage_get_tracing,
 413         .gpl_only       = false,
 414         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 415         .arg1_type      = ARG_CONST_MAP_PTR,
 416         .arg2_type      = ARG_PTR_TO_BTF_ID_OR_NULL,
 417         .arg2_btf_id    = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
 418         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 419         .arg4_type      = ARG_ANYTHING,
 420         .allowed        = bpf_sk_storage_tracing_allowed,
 421 };
 422
 423 const struct bpf_func_proto bpf_sk_storage_delete_tracing_proto = {
 424         .func           = bpf_sk_storage_delete_tracing,
 425         .gpl_only       = false,
 426         .ret_type       = RET_INTEGER,
 427         .arg1_type      = ARG_CONST_MAP_PTR,
 428         .arg2_type      = ARG_PTR_TO_BTF_ID_OR_NULL,
 429         .arg2_btf_id    = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
 430         .allowed        = bpf_sk_storage_tracing_allowed,
 431 };
 432
 433 struct bpf_sk_storage_diag {
 434         u32 nr_maps;
 435         struct bpf_map *maps[];
 436 };
 437
 438 /* The reply will be like:
 439  * INET_DIAG_BPF_SK_STORAGES (nla_nest)
 440  *      SK_DIAG_BPF_STORAGE (nla_nest)
 441  *              SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
 442  *              SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
 443  *      SK_DIAG_BPF_STORAGE (nla_nest)
 444  *              SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
 445  *              SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
 446  *      ....
 447  */
 448 static int nla_value_size(u32 value_size)
 449 {
 450         /* SK_DIAG_BPF_STORAGE (nla_nest)
 451          *      SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
 452          *      SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
 453          */
 454         return nla_total_size(0) + nla_total_size(sizeof(u32)) +
 455                 nla_total_size_64bit(value_size);
 456 }
 457
 458 void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag)
 459 {
 460         u32 i;
 461
 462         if (!diag)
 463                 return;
 464
 465         for (i = 0; i < diag->nr_maps; i++)
 466                 bpf_map_put(diag->maps[i]);
 467
 468         kfree(diag);
 469 }
 470 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free);
 471
 472 static bool diag_check_dup(const struct bpf_sk_storage_diag *diag,
 473                            const struct bpf_map *map)
 474 {
 475         u32 i;
 476
 477         for (i = 0; i < diag->nr_maps; i++) {
 478                 if (diag->maps[i] == map)
 479                         return true;
 480         }
 481
 482         return false;
 483 }
 484
 485 struct bpf_sk_storage_diag *
 486 bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs)
 487 {
 488         struct bpf_sk_storage_diag *diag;
 489         struct nlattr *nla;
 490         u32 nr_maps = 0;
 491         int rem, err;
 492
 493         /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as
 494          * the map_alloc_check() side also does.
 495          */
 496         if (!bpf_capable())
 497                 return ERR_PTR(-EPERM);
 498
 499         nla_for_each_nested(nla, nla_stgs, rem) {
 500                 if (nla_type(nla) == SK_DIAG_BPF_STORAGE_REQ_MAP_FD) {
 501                         if (nla_len(nla) != sizeof(u32))
 502                                 return ERR_PTR(-EINVAL);
 503                         nr_maps++;
 504                 }
 505         }
 506
 507         diag = kzalloc(struct_size(diag, maps, nr_maps), GFP_KERNEL);
 508         if (!diag)
 509                 return ERR_PTR(-ENOMEM);
 510
 511         nla_for_each_nested(nla, nla_stgs, rem) {
 512                 struct bpf_map *map;
 513                 int map_fd;
 514
 515                 if (nla_type(nla) != SK_DIAG_BPF_STORAGE_REQ_MAP_FD)
 516                         continue;
 517
 518                 map_fd = nla_get_u32(nla);
 519                 map = bpf_map_get(map_fd);
 520                 if (IS_ERR(map)) {
 521                         err = PTR_ERR(map);
 522                         goto err_free;
 523                 }
 524                 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) {
 525                         bpf_map_put(map);
 526                         err = -EINVAL;
 527                         goto err_free;
 528                 }
 529                 if (diag_check_dup(diag, map)) {
 530                         bpf_map_put(map);
 531                         err = -EEXIST;
 532                         goto err_free;
 533                 }
 534                 diag->maps[diag->nr_maps++] = map;
 535         }
 536
 537         return diag;
 538
 539 err_free:
 540         bpf_sk_storage_diag_free(diag);
 541         return ERR_PTR(err);
 542 }
 543 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
 544
 545 static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
 546 {
 547         struct nlattr *nla_stg, *nla_value;
 548         struct bpf_local_storage_map *smap;
 549
 550         /* It cannot exceed max nlattr's payload */
 551         BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
 552
 553         nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE);
 554         if (!nla_stg)
 555                 return -EMSGSIZE;
 556
 557         smap = rcu_dereference(sdata->smap);
 558         if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
 559                 goto errout;
 560
 561         nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE,
 562                                       smap->map.value_size,
 563                                       SK_DIAG_BPF_STORAGE_PAD);
 564         if (!nla_value)
 565                 goto errout;
 566
 567         if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK))
 568                 copy_map_value_locked(&smap->map, nla_data(nla_value),
 569                                       sdata->data, true);
 570         else
 571                 copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
 572
 573         nla_nest_end(skb, nla_stg);
 574         return 0;
 575
 576 errout:
 577         nla_nest_cancel(skb, nla_stg);
 578         return -EMSGSIZE;
 579 }
 580
 581 static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
 582                                        int stg_array_type,
 583                                        unsigned int *res_diag_size)
 584 {
 585         /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
 586         unsigned int diag_size = nla_total_size(0);
 587         struct bpf_local_storage *sk_storage;
 588         struct bpf_local_storage_elem *selem;
 589         struct bpf_local_storage_map *smap;
 590         struct nlattr *nla_stgs;
 591         unsigned int saved_len;
 592         int err = 0;
 593
 594         rcu_read_lock();
 595
 596         sk_storage = rcu_dereference(sk->sk_bpf_storage);
 597         if (!sk_storage || hlist_empty(&sk_storage->list)) {
 598                 rcu_read_unlock();
 599                 return 0;
 600         }
 601
 602         nla_stgs = nla_nest_start(skb, stg_array_type);
 603         if (!nla_stgs)
 604                 /* Continue to learn diag_size */
 605                 err = -EMSGSIZE;
 606
 607         saved_len = skb->len;
 608         hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
 609                 smap = rcu_dereference(SDATA(selem)->smap);
 610                 diag_size += nla_value_size(smap->map.value_size);
 611
 612                 if (nla_stgs && diag_get(SDATA(selem), skb))
 613                         /* Continue to learn diag_size */
 614                         err = -EMSGSIZE;
 615         }
 616
 617         rcu_read_unlock();
 618
 619         if (nla_stgs) {
 620                 if (saved_len == skb->len)
 621                         nla_nest_cancel(skb, nla_stgs);
 622                 else
 623                         nla_nest_end(skb, nla_stgs);
 624         }
 625
 626         if (diag_size == nla_total_size(0)) {
 627                 *res_diag_size = 0;
 628                 return 0;
 629         }
 630
 631         *res_diag_size = diag_size;
 632         return err;
 633 }
 634
 635 int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
 636                             struct sock *sk, struct sk_buff *skb,
 637                             int stg_array_type,
 638                             unsigned int *res_diag_size)
 639 {
 640         /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
 641         unsigned int diag_size = nla_total_size(0);
 642         struct bpf_local_storage *sk_storage;
 643         struct bpf_local_storage_data *sdata;
 644         struct nlattr *nla_stgs;
 645         unsigned int saved_len;
 646         int err = 0;
 647         u32 i;
 648
 649         *res_diag_size = 0;
 650
 651         /* No map has been specified.  Dump all. */
 652         if (!diag->nr_maps)
 653                 return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type,
 654                                                    res_diag_size);
 655
 656         rcu_read_lock();
 657         sk_storage = rcu_dereference(sk->sk_bpf_storage);
 658         if (!sk_storage || hlist_empty(&sk_storage->list)) {
 659                 rcu_read_unlock();
 660                 return 0;
 661         }
 662
 663         nla_stgs = nla_nest_start(skb, stg_array_type);
 664         if (!nla_stgs)
 665                 /* Continue to learn diag_size */
 666                 err = -EMSGSIZE;
 667
 668         saved_len = skb->len;
 669         for (i = 0; i < diag->nr_maps; i++) {
 670                 sdata = bpf_local_storage_lookup(sk_storage,
 671                                 (struct bpf_local_storage_map *)diag->maps[i],
 672                                 false);
 673
 674                 if (!sdata)
 675                         continue;
 676
 677                 diag_size += nla_value_size(diag->maps[i]->value_size);
 678
 679                 if (nla_stgs && diag_get(sdata, skb))
 680                         /* Continue to learn diag_size */
 681                         err = -EMSGSIZE;
 682         }
 683         rcu_read_unlock();
 684
 685         if (nla_stgs) {
 686                 if (saved_len == skb->len)
 687                         nla_nest_cancel(skb, nla_stgs);
 688                 else
 689                         nla_nest_end(skb, nla_stgs);
 690         }
 691
 692         if (diag_size == nla_total_size(0)) {
 693                 *res_diag_size = 0;
 694                 return 0;
 695         }
 696
 697         *res_diag_size = diag_size;
 698         return err;
 699 }
 700 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put);
 701
 702 struct bpf_iter_seq_sk_storage_map_info {
 703         struct bpf_map *map;
 704         unsigned int bucket_id;
 705         unsigned skip_elems;
 706 };
 707
 708 static struct bpf_local_storage_elem *
 709 bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info,
 710                                  struct bpf_local_storage_elem *prev_selem)
 711         __acquires(RCU) __releases(RCU)
 712 {
 713         struct bpf_local_storage *sk_storage;
 714         struct bpf_local_storage_elem *selem;
 715         u32 skip_elems = info->skip_elems;
 716         struct bpf_local_storage_map *smap;
 717         u32 bucket_id = info->bucket_id;
 718         u32 i, count, n_buckets;
 719         struct bpf_local_storage_map_bucket *b;
 720
 721         smap = (struct bpf_local_storage_map *)info->map;
 722         n_buckets = 1U << smap->bucket_log;
 723         if (bucket_id >= n_buckets)
 724                 return NULL;
 725
 726         /* try to find next selem in the same bucket */
 727         selem = prev_selem;
 728         count = 0;
 729         while (selem) {
 730                 selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)),
 731                                          struct bpf_local_storage_elem, map_node);
 732                 if (!selem) {
 733                         /* not found, unlock and go to the next bucket */
 734                         b = &smap->buckets[bucket_id++];
 735                         rcu_read_unlock();
 736                         skip_elems = 0;
 737                         break;
 738                 }
 739                 sk_storage = rcu_dereference(selem->local_storage);
 740                 if (sk_storage) {
 741                         info->skip_elems = skip_elems + count;
 742                         return selem;
 743                 }
 744                 count++;
 745         }
 746
 747         for (i = bucket_id; i < (1U << smap->bucket_log); i++) {
 748                 b = &smap->buckets[i];
 749                 rcu_read_lock();
 750                 count = 0;
 751                 hlist_for_each_entry_rcu(selem, &b->list, map_node) {
 752                         sk_storage = rcu_dereference(selem->local_storage);
 753                         if (sk_storage && count >= skip_elems) {
 754                                 info->bucket_id = i;
 755                                 info->skip_elems = count;
 756                                 return selem;
 757                         }
 758                         count++;
 759                 }
 760                 rcu_read_unlock();
 761                 skip_elems = 0;
 762         }
 763
 764         info->bucket_id = i;
 765         info->skip_elems = 0;
 766         return NULL;
 767 }
 768
 769 static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos)
 770 {
 771         struct bpf_local_storage_elem *selem;
 772
 773         selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL);
 774         if (!selem)
 775                 return NULL;
 776
 777         if (*pos == 0)
 778                 ++*pos;
 779         return selem;
 780 }
 781
 782 static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v,
 783                                          loff_t *pos)
 784 {
 785         struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
 786
 787         ++*pos;
 788         ++info->skip_elems;
 789         return bpf_sk_storage_map_seq_find_next(seq->private, v);
 790 }
 791
 792 struct bpf_iter__bpf_sk_storage_map {
 793         __bpf_md_ptr(struct bpf_iter_meta *, meta);
 794         __bpf_md_ptr(struct bpf_map *, map);
 795         __bpf_md_ptr(struct sock *, sk);
 796         __bpf_md_ptr(void *, value);
 797 };
 798
 799 DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta,
 800                      struct bpf_map *map, struct sock *sk,
 801                      void *value)
 802
 803 static int __bpf_sk_storage_map_seq_show(struct seq_file *seq,
 804                                          struct bpf_local_storage_elem *selem)
 805 {
 806         struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
 807         struct bpf_iter__bpf_sk_storage_map ctx = {};
 808         struct bpf_local_storage *sk_storage;
 809         struct bpf_iter_meta meta;
 810         struct bpf_prog *prog;
 811         int ret = 0;
 812
 813         meta.seq = seq;
 814         prog = bpf_iter_get_info(&meta, selem == NULL);
 815         if (prog) {
 816                 ctx.meta = &meta;
 817                 ctx.map = info->map;
 818                 if (selem) {
 819                         sk_storage = rcu_dereference(selem->local_storage);
 820                         ctx.sk = sk_storage->owner;
 821                         ctx.value = SDATA(selem)->data;
 822                 }
 823                 ret = bpf_iter_run_prog(prog, &ctx);
 824         }
 825
 826         return ret;
 827 }
 828
 829 static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v)
 830 {
 831         return __bpf_sk_storage_map_seq_show(seq, v);
 832 }
 833
 834 static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v)
 835         __releases(RCU)
 836 {
 837         if (!v)
 838                 (void)__bpf_sk_storage_map_seq_show(seq, v);
 839         else
 840                 rcu_read_unlock();
 841 }
 842
 843 static int bpf_iter_init_sk_storage_map(void *priv_data,
 844                                         struct bpf_iter_aux_info *aux)
 845 {
 846         struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
 847
 848         bpf_map_inc_with_uref(aux->map);
 849         seq_info->map = aux->map;
 850         return 0;
 851 }
 852
 853 static void bpf_iter_fini_sk_storage_map(void *priv_data)
 854 {
 855         struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
 856
 857         bpf_map_put_with_uref(seq_info->map);
 858 }
 859
 860 static int bpf_iter_attach_map(struct bpf_prog *prog,
 861                                union bpf_iter_link_info *linfo,
 862                                struct bpf_iter_aux_info *aux)
 863 {
 864         struct bpf_map *map;
 865         int err = -EINVAL;
 866
 867         if (!linfo->map.map_fd)
 868                 return -EBADF;
 869
 870         map = bpf_map_get_with_uref(linfo->map.map_fd);
 871         if (IS_ERR(map))
 872                 return PTR_ERR(map);
 873
 874         if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
 875                 goto put_map;
 876
 877         if (prog->aux->max_rdwr_access > map->value_size) {
 878                 err = -EACCES;
 879                 goto put_map;
 880         }
 881
 882         aux->map = map;
 883         return 0;
 884
 885 put_map:
 886         bpf_map_put_with_uref(map);
 887         return err;
 888 }
 889
 890 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux)
 891 {
 892         bpf_map_put_with_uref(aux->map);
 893 }
 894
 895 static const struct seq_operations bpf_sk_storage_map_seq_ops = {
 896         .start  = bpf_sk_storage_map_seq_start,
 897         .next   = bpf_sk_storage_map_seq_next,
 898         .stop   = bpf_sk_storage_map_seq_stop,
 899         .show   = bpf_sk_storage_map_seq_show,
 900 };
 901
 902 static const struct bpf_iter_seq_info iter_seq_info = {
 903         .seq_ops                = &bpf_sk_storage_map_seq_ops,
 904         .init_seq_private       = bpf_iter_init_sk_storage_map,
 905         .fini_seq_private       = bpf_iter_fini_sk_storage_map,
 906         .seq_priv_size          = sizeof(struct bpf_iter_seq_sk_storage_map_info),
 907 };
 908
 909 static struct bpf_iter_reg bpf_sk_storage_map_reg_info = {
 910         .target                 = "bpf_sk_storage_map",
 911         .attach_target          = bpf_iter_attach_map,
 912         .detach_target          = bpf_iter_detach_map,
 913         .show_fdinfo            = bpf_iter_map_show_fdinfo,
 914         .fill_link_info         = bpf_iter_map_fill_link_info,
 915         .ctx_arg_info_size      = 2,
 916         .ctx_arg_info           = {
 917                 { offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
 918                   PTR_TO_BTF_ID_OR_NULL },
 919                 { offsetof(struct bpf_iter__bpf_sk_storage_map, value),
 920                   PTR_TO_BUF | PTR_MAYBE_NULL },
 921         },
 922         .seq_info               = &iter_seq_info,
 923 };
 924
 925 static int __init bpf_sk_storage_map_iter_init(void)
 926 {
 927         bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id =
 928                 btf_sock_ids[BTF_SOCK_TYPE_SOCK];
 929         return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info);
 930 }
 931 late_initcall(bpf_sk_storage_map_iter_init);