1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/mutex.h>
4 #include <linux/netdevice.h>
5 #include <linux/xarray.h>
6 #include <net/net_debug.h>
7 #include <net/page_pool/types.h>
8 #include <net/page_pool/helpers.h>
11 #include "page_pool_priv.h"
12 #include "netdev-genl-gen.h"
14 static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1);
15 /* Protects: page_pools, netdevice->page_pools, pool->slow.netdev, pool->user.
16 * Ordering: inside rtnl_lock
18 static DEFINE_MUTEX(page_pools_lock);
20 /* Page pools are only reachable from user space (via netlink) if they are
21 * linked to a netdev at creation time. Following page pool "visibility"
22 * states are possible:
24 * - user.list: linked to real netdev, netdev: real netdev
25 * - orphaned - real netdev has disappeared
26 * - user.list: linked to lo, netdev: lo
27 * - invisible - either (a) created without netdev linking, (b) unlisted due
28 * to error, or (c) the entire namespace which owned this pool disappeared
29 * - user.list: unhashed, netdev: unknown
32 typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool,
33 const struct genl_info *info);
36 netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill)
38 struct page_pool *pool;
42 mutex_lock(&page_pools_lock);
43 pool = xa_load(&page_pools, id);
44 if (!pool || hlist_unhashed(&pool->user.list) ||
45 !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) {
50 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
56 err = fill(rsp, pool, info);
60 mutex_unlock(&page_pools_lock);
62 return genlmsg_reply(rsp, info);
67 mutex_unlock(&page_pools_lock);
71 struct page_pool_dump_cb {
72 unsigned long ifindex;
77 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
80 struct page_pool_dump_cb *state = (void *)cb->ctx;
81 const struct genl_info *info = genl_info_dump(cb);
82 struct net *net = sock_net(skb->sk);
83 struct net_device *netdev;
84 struct page_pool *pool;
88 mutex_lock(&page_pools_lock);
89 for_each_netdev_dump(net, netdev, state->ifindex) {
90 hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
91 if (state->pp_id && state->pp_id < pool->user.id)
94 state->pp_id = pool->user.id;
95 err = fill(skb, pool, info);
103 mutex_unlock(&page_pools_lock);
110 page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool,
111 const struct genl_info *info)
113 #ifdef CONFIG_PAGE_POOL_STATS
114 struct page_pool_stats stats = {};
118 if (!page_pool_get_stats(pool, &stats))
121 hdr = genlmsg_iput(rsp, info);
125 nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO);
127 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) ||
128 (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
129 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
130 pool->slow.netdev->ifindex)))
131 goto err_cancel_nest;
133 nla_nest_end(rsp, nest);
135 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST,
136 stats.alloc_stats.fast) ||
137 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW,
138 stats.alloc_stats.slow) ||
139 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER,
140 stats.alloc_stats.slow_high_order) ||
141 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY,
142 stats.alloc_stats.empty) ||
143 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL,
144 stats.alloc_stats.refill) ||
145 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE,
146 stats.alloc_stats.waive) ||
147 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED,
148 stats.recycle_stats.cached) ||
149 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL,
150 stats.recycle_stats.cache_full) ||
151 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING,
152 stats.recycle_stats.ring) ||
153 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL,
154 stats.recycle_stats.ring_full) ||
155 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT,
156 stats.recycle_stats.released_refcnt))
159 genlmsg_end(rsp, hdr);
163 nla_nest_cancel(rsp, nest);
165 genlmsg_cancel(rsp, hdr);
168 GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS");
173 int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb,
174 struct genl_info *info)
176 struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)];
181 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO))
184 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];
185 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest,
186 netdev_page_pool_info_nl_policy,
191 if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID))
193 if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) {
194 NL_SET_ERR_MSG_ATTR(info->extack,
195 tb[NETDEV_A_PAGE_POOL_IFINDEX],
196 "selecting by ifindex not supported");
200 id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]);
202 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill);
205 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb,
206 struct netlink_callback *cb)
208 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill);
212 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool,
213 const struct genl_info *info)
215 size_t inflight, refsz;
218 hdr = genlmsg_iput(rsp, info);
222 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id))
225 if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
226 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
227 pool->slow.netdev->ifindex))
229 if (pool->user.napi_id &&
230 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, pool->user.napi_id))
233 inflight = page_pool_inflight(pool, false);
234 refsz = PAGE_SIZE << pool->p.order;
235 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) ||
236 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM,
239 if (pool->user.detach_time &&
240 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME,
241 pool->user.detach_time))
244 genlmsg_end(rsp, hdr);
248 genlmsg_cancel(rsp, hdr);
252 static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd)
254 struct genl_info info;
258 lockdep_assert_held(&page_pools_lock);
260 /* 'invisible' page pools don't matter */
261 if (hlist_unhashed(&pool->user.list))
263 net = dev_net(pool->slow.netdev);
265 if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL))
268 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
270 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
274 if (page_pool_nl_fill(ntf, pool, &info)) {
279 genlmsg_multicast_netns(&netdev_nl_family, net, ntf,
280 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL);
283 int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info)
287 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID))
290 id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]);
292 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill);
295 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb,
296 struct netlink_callback *cb)
298 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill);
301 int page_pool_list(struct page_pool *pool)
303 static u32 id_alloc_next;
306 mutex_lock(&page_pools_lock);
307 err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b,
308 &id_alloc_next, GFP_KERNEL);
312 INIT_HLIST_NODE(&pool->user.list);
313 if (pool->slow.netdev) {
314 hlist_add_head(&pool->user.list,
315 &pool->slow.netdev->page_pools);
316 pool->user.napi_id = pool->p.napi ? pool->p.napi->napi_id : 0;
318 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF);
321 mutex_unlock(&page_pools_lock);
325 mutex_unlock(&page_pools_lock);
329 void page_pool_detached(struct page_pool *pool)
331 mutex_lock(&page_pools_lock);
332 pool->user.detach_time = ktime_get_boottime_seconds();
333 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
334 mutex_unlock(&page_pools_lock);
337 void page_pool_unlist(struct page_pool *pool)
339 mutex_lock(&page_pools_lock);
340 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF);
341 xa_erase(&page_pools, pool->user.id);
342 if (!hlist_unhashed(&pool->user.list))
343 hlist_del(&pool->user.list);
344 mutex_unlock(&page_pools_lock);
347 static void page_pool_unreg_netdev_wipe(struct net_device *netdev)
349 struct page_pool *pool;
350 struct hlist_node *n;
352 mutex_lock(&page_pools_lock);
353 hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) {
354 hlist_del_init(&pool->user.list);
355 pool->slow.netdev = NET_PTR_POISON;
357 mutex_unlock(&page_pools_lock);
360 static void page_pool_unreg_netdev(struct net_device *netdev)
362 struct page_pool *pool, *last;
363 struct net_device *lo;
365 lo = dev_net(netdev)->loopback_dev;
367 mutex_lock(&page_pools_lock);
369 hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
370 pool->slow.netdev = lo;
371 netdev_nl_page_pool_event(pool,
372 NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
376 hlist_splice_init(&netdev->page_pools, &last->user.list,
378 mutex_unlock(&page_pools_lock);
382 page_pool_netdevice_event(struct notifier_block *nb,
383 unsigned long event, void *ptr)
385 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
387 if (event != NETDEV_UNREGISTER)
390 if (hlist_empty(&netdev->page_pools))
393 if (netdev->ifindex != LOOPBACK_IFINDEX)
394 page_pool_unreg_netdev(netdev);
396 page_pool_unreg_netdev_wipe(netdev);
400 static struct notifier_block page_pool_netdevice_nb = {
401 .notifier_call = page_pool_netdevice_event,
404 static int __init page_pool_user_init(void)
406 return register_netdevice_notifier(&page_pool_netdevice_nb);
409 subsys_initcall(page_pool_user_init);