core/page_pool_user.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/mutex.h>
   4 #include <linux/netdevice.h>
   5 #include <linux/xarray.h>
   6 #include <net/net_debug.h>
   7 #include <net/page_pool/types.h>
   8 #include <net/page_pool/helpers.h>
   9 #include <net/sock.h>
  10
  11 #include "page_pool_priv.h"
  12 #include "netdev-genl-gen.h"
  13
  14 static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1);
  15 /* Protects: page_pools, netdevice->page_pools, pool->slow.netdev, pool->user.
  16  * Ordering: inside rtnl_lock
  17  */
  18 static DEFINE_MUTEX(page_pools_lock);
  19
  20 /* Page pools are only reachable from user space (via netlink) if they are
  21  * linked to a netdev at creation time. Following page pool "visibility"
  22  * states are possible:
  23  *  - normal
  24  *    - user.list: linked to real netdev, netdev: real netdev
  25  *  - orphaned - real netdev has disappeared
  26  *    - user.list: linked to lo, netdev: lo
  27  *  - invisible - either (a) created without netdev linking, (b) unlisted due
  28  *      to error, or (c) the entire namespace which owned this pool disappeared
  29  *    - user.list: unhashed, netdev: unknown
  30  */
  31
  32 typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool,
  33                              const struct genl_info *info);
  34
  35 static int
  36 netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill)
  37 {
  38         struct page_pool *pool;
  39         struct sk_buff *rsp;
  40         int err;
  41
  42         mutex_lock(&page_pools_lock);
  43         pool = xa_load(&page_pools, id);
  44         if (!pool || hlist_unhashed(&pool->user.list) ||
  45             !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) {
  46                 err = -ENOENT;
  47                 goto err_unlock;
  48         }
  49
  50         rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
  51         if (!rsp) {
  52                 err = -ENOMEM;
  53                 goto err_unlock;
  54         }
  55
  56         err = fill(rsp, pool, info);
  57         if (err)
  58                 goto err_free_msg;
  59
  60         mutex_unlock(&page_pools_lock);
  61
  62         return genlmsg_reply(rsp, info);
  63
  64 err_free_msg:
  65         nlmsg_free(rsp);
  66 err_unlock:
  67         mutex_unlock(&page_pools_lock);
  68         return err;
  69 }
  70
  71 struct page_pool_dump_cb {
  72         unsigned long ifindex;
  73         u32 pp_id;
  74 };
  75
  76 static int
  77 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
  78                              pp_nl_fill_cb fill)
  79 {
  80         struct page_pool_dump_cb *state = (void *)cb->ctx;
  81         const struct genl_info *info = genl_info_dump(cb);
  82         struct net *net = sock_net(skb->sk);
  83         struct net_device *netdev;
  84         struct page_pool *pool;
  85         int err = 0;
  86
  87         rtnl_lock();
  88         mutex_lock(&page_pools_lock);
  89         for_each_netdev_dump(net, netdev, state->ifindex) {
  90                 hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
  91                         if (state->pp_id && state->pp_id < pool->user.id)
  92                                 continue;
  93
  94                         state->pp_id = pool->user.id;
  95                         err = fill(skb, pool, info);
  96                         if (err)
  97                                 goto out;
  98                 }
  99
 100                 state->pp_id = 0;
 101         }
 102 out:
 103         mutex_unlock(&page_pools_lock);
 104         rtnl_unlock();
 105
 106         return err;
 107 }
 108
 109 static int
 110 page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool,
 111                         const struct genl_info *info)
 112 {
 113 #ifdef CONFIG_PAGE_POOL_STATS
 114         struct page_pool_stats stats = {};
 115         struct nlattr *nest;
 116         void *hdr;
 117
 118         if (!page_pool_get_stats(pool, &stats))
 119                 return 0;
 120
 121         hdr = genlmsg_iput(rsp, info);
 122         if (!hdr)
 123                 return -EMSGSIZE;
 124
 125         nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO);
 126
 127         if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) ||
 128             (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
 129              nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
 130                          pool->slow.netdev->ifindex)))
 131                 goto err_cancel_nest;
 132
 133         nla_nest_end(rsp, nest);
 134
 135         if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST,
 136                          stats.alloc_stats.fast) ||
 137             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW,
 138                          stats.alloc_stats.slow) ||
 139             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER,
 140                          stats.alloc_stats.slow_high_order) ||
 141             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY,
 142                          stats.alloc_stats.empty) ||
 143             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL,
 144                          stats.alloc_stats.refill) ||
 145             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE,
 146                          stats.alloc_stats.waive) ||
 147             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED,
 148                          stats.recycle_stats.cached) ||
 149             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL,
 150                          stats.recycle_stats.cache_full) ||
 151             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING,
 152                          stats.recycle_stats.ring) ||
 153             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL,
 154                          stats.recycle_stats.ring_full) ||
 155             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT,
 156                          stats.recycle_stats.released_refcnt))
 157                 goto err_cancel_msg;
 158
 159         genlmsg_end(rsp, hdr);
 160
 161         return 0;
 162 err_cancel_nest:
 163         nla_nest_cancel(rsp, nest);
 164 err_cancel_msg:
 165         genlmsg_cancel(rsp, hdr);
 166         return -EMSGSIZE;
 167 #else
 168         GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS");
 169         return -EOPNOTSUPP;
 170 #endif
 171 }
 172
 173 int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb,
 174                                        struct genl_info *info)
 175 {
 176         struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)];
 177         struct nlattr *nest;
 178         int err;
 179         u32 id;
 180
 181         if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO))
 182                 return -EINVAL;
 183
 184         nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];
 185         err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest,
 186                                netdev_page_pool_info_nl_policy,
 187                                info->extack);
 188         if (err)
 189                 return err;
 190
 191         if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID))
 192                 return -EINVAL;
 193         if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) {
 194                 NL_SET_ERR_MSG_ATTR(info->extack,
 195                                     tb[NETDEV_A_PAGE_POOL_IFINDEX],
 196                                     "selecting by ifindex not supported");
 197                 return -EINVAL;
 198         }
 199
 200         id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]);
 201
 202         return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill);
 203 }
 204
 205 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb,
 206                                          struct netlink_callback *cb)
 207 {
 208         return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill);
 209 }
 210
 211 static int
 212 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool,
 213                   const struct genl_info *info)
 214 {
 215         size_t inflight, refsz;
 216         void *hdr;
 217
 218         hdr = genlmsg_iput(rsp, info);
 219         if (!hdr)
 220                 return -EMSGSIZE;
 221
 222         if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id))
 223                 goto err_cancel;
 224
 225         if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
 226             nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
 227                         pool->slow.netdev->ifindex))
 228                 goto err_cancel;
 229         if (pool->user.napi_id &&
 230             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, pool->user.napi_id))
 231                 goto err_cancel;
 232
 233         inflight = page_pool_inflight(pool, false);
 234         refsz = PAGE_SIZE << pool->p.order;
 235         if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) ||
 236             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM,
 237                          inflight * refsz))
 238                 goto err_cancel;
 239         if (pool->user.detach_time &&
 240             nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME,
 241                          pool->user.detach_time))
 242                 goto err_cancel;
 243
 244         genlmsg_end(rsp, hdr);
 245
 246         return 0;
 247 err_cancel:
 248         genlmsg_cancel(rsp, hdr);
 249         return -EMSGSIZE;
 250 }
 251
 252 static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd)
 253 {
 254         struct genl_info info;
 255         struct sk_buff *ntf;
 256         struct net *net;
 257
 258         lockdep_assert_held(&page_pools_lock);
 259
 260         /* 'invisible' page pools don't matter */
 261         if (hlist_unhashed(&pool->user.list))
 262                 return;
 263         net = dev_net(pool->slow.netdev);
 264
 265         if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL))
 266                 return;
 267
 268         genl_info_init_ntf(&info, &netdev_nl_family, cmd);
 269
 270         ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
 271         if (!ntf)
 272                 return;
 273
 274         if (page_pool_nl_fill(ntf, pool, &info)) {
 275                 nlmsg_free(ntf);
 276                 return;
 277         }
 278
 279         genlmsg_multicast_netns(&netdev_nl_family, net, ntf,
 280                                 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL);
 281 }
 282
 283 int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info)
 284 {
 285         u32 id;
 286
 287         if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID))
 288                 return -EINVAL;
 289
 290         id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]);
 291
 292         return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill);
 293 }
 294
 295 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb,
 296                                    struct netlink_callback *cb)
 297 {
 298         return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill);
 299 }
 300
 301 int page_pool_list(struct page_pool *pool)
 302 {
 303         static u32 id_alloc_next;
 304         int err;
 305
 306         mutex_lock(&page_pools_lock);
 307         err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b,
 308                               &id_alloc_next, GFP_KERNEL);
 309         if (err < 0)
 310                 goto err_unlock;
 311
 312         INIT_HLIST_NODE(&pool->user.list);
 313         if (pool->slow.netdev) {
 314                 hlist_add_head(&pool->user.list,
 315                                &pool->slow.netdev->page_pools);
 316                 pool->user.napi_id = pool->p.napi ? pool->p.napi->napi_id : 0;
 317
 318                 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF);
 319         }
 320
 321         mutex_unlock(&page_pools_lock);
 322         return 0;
 323
 324 err_unlock:
 325         mutex_unlock(&page_pools_lock);
 326         return err;
 327 }
 328
 329 void page_pool_detached(struct page_pool *pool)
 330 {
 331         mutex_lock(&page_pools_lock);
 332         pool->user.detach_time = ktime_get_boottime_seconds();
 333         netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
 334         mutex_unlock(&page_pools_lock);
 335 }
 336
 337 void page_pool_unlist(struct page_pool *pool)
 338 {
 339         mutex_lock(&page_pools_lock);
 340         netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF);
 341         xa_erase(&page_pools, pool->user.id);
 342         if (!hlist_unhashed(&pool->user.list))
 343                 hlist_del(&pool->user.list);
 344         mutex_unlock(&page_pools_lock);
 345 }
 346
 347 static void page_pool_unreg_netdev_wipe(struct net_device *netdev)
 348 {
 349         struct page_pool *pool;
 350         struct hlist_node *n;
 351
 352         mutex_lock(&page_pools_lock);
 353         hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) {
 354                 hlist_del_init(&pool->user.list);
 355                 pool->slow.netdev = NET_PTR_POISON;
 356         }
 357         mutex_unlock(&page_pools_lock);
 358 }
 359
 360 static void page_pool_unreg_netdev(struct net_device *netdev)
 361 {
 362         struct page_pool *pool, *last;
 363         struct net_device *lo;
 364
 365         lo = dev_net(netdev)->loopback_dev;
 366
 367         mutex_lock(&page_pools_lock);
 368         last = NULL;
 369         hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
 370                 pool->slow.netdev = lo;
 371                 netdev_nl_page_pool_event(pool,
 372                                           NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
 373                 last = pool;
 374         }
 375         if (last)
 376                 hlist_splice_init(&netdev->page_pools, &last->user.list,
 377                                   &lo->page_pools);
 378         mutex_unlock(&page_pools_lock);
 379 }
 380
 381 static int
 382 page_pool_netdevice_event(struct notifier_block *nb,
 383                           unsigned long event, void *ptr)
 384 {
 385         struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
 386
 387         if (event != NETDEV_UNREGISTER)
 388                 return NOTIFY_DONE;
 389
 390         if (hlist_empty(&netdev->page_pools))
 391                 return NOTIFY_OK;
 392
 393         if (netdev->ifindex != LOOPBACK_IFINDEX)
 394                 page_pool_unreg_netdev(netdev);
 395         else
 396                 page_pool_unreg_netdev_wipe(netdev);
 397         return NOTIFY_OK;
 398 }
 399
 400 static struct notifier_block page_pool_netdevice_nb = {
 401         .notifier_call = page_pool_netdevice_event,
 402 };
 403
 404 static int __init page_pool_user_init(void)
 405 {
 406         return register_netdevice_notifier(&page_pool_netdevice_nb);
 407 }
 408
 409 subsys_initcall(page_pool_user_init);