1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
3 * Copyright (C) 2018 Netronome Systems, Inc.
5 * This software is dual licensed under the GNU General License Version 2,
6 * June 1991 as shown in the file COPYING in the top-level directory of this
7 * source tree or the BSD 2-Clause License provided below. You have the
8 * option to license this software under the complete terms of either license.
10 * The BSD 2-Clause License:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * 1. Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * 2. Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/bitfield.h>
36 #include <linux/etherdevice.h>
37 #include <linux/lockdep.h>
38 #include <linux/netdevice.h>
39 #include <linux/rcupdate.h>
40 #include <linux/slab.h>
41 #include <net/pkt_cls.h>
42 #include <net/pkt_sched.h>
45 #include "../nfpcore/nfp.h"
46 #include "../nfpcore/nfp_cpp.h"
47 #include "../nfpcore/nfp_nsp.h"
48 #include "../nfp_app.h"
49 #include "../nfp_main.h"
50 #include "../nfp_net.h"
51 #include "../nfp_net_repr.h"
52 #include "../nfp_port.h"
55 static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id)
57 return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) |
58 FIELD_PREP(NFP_ABM_PORTID_ID, id);
62 __nfp_abm_reset_root(struct net_device *netdev, struct nfp_abm_link *alink,
63 u32 handle, unsigned int qs, u32 init_val)
65 struct nfp_port *port = nfp_port_from_netdev(netdev);
68 ret = nfp_abm_ctrl_set_all_q_lvls(alink, init_val);
69 memset(alink->qdiscs, 0, sizeof(*alink->qdiscs) * alink->num_qdiscs);
71 alink->parent = handle;
72 alink->num_qdiscs = qs;
73 port->tc_offload_cnt = qs;
79 nfp_abm_reset_root(struct net_device *netdev, struct nfp_abm_link *alink,
80 u32 handle, unsigned int qs)
82 __nfp_abm_reset_root(netdev, alink, handle, qs, ~0);
86 nfp_abm_red_find(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt)
88 unsigned int i = TC_H_MIN(opt->parent) - 1;
90 if (opt->parent == TC_H_ROOT)
92 else if (TC_H_MAJ(alink->parent) == TC_H_MAJ(opt->parent))
93 i = TC_H_MIN(opt->parent) - 1;
97 if (i >= alink->num_qdiscs || opt->handle != alink->qdiscs[i].handle)
104 nfp_abm_red_destroy(struct net_device *netdev, struct nfp_abm_link *alink,
109 for (i = 0; i < alink->num_qdiscs; i++)
110 if (handle == alink->qdiscs[i].handle)
112 if (i == alink->num_qdiscs)
115 if (alink->parent == TC_H_ROOT) {
116 nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 0);
118 nfp_abm_ctrl_set_q_lvl(alink, i, ~0);
119 memset(&alink->qdiscs[i], 0, sizeof(*alink->qdiscs));
124 nfp_abm_red_replace(struct net_device *netdev, struct nfp_abm_link *alink,
125 struct tc_red_qopt_offload *opt)
130 i = nfp_abm_red_find(alink, opt);
133 if (opt->set.min != opt->set.max || !opt->set.is_ecn) {
134 nfp_warn(alink->abm->app->cpp,
135 "RED offload failed - unsupported parameters\n");
141 if (alink->parent == TC_H_ROOT)
142 err = nfp_abm_ctrl_set_all_q_lvls(alink, opt->set.min);
144 err = nfp_abm_ctrl_set_q_lvl(alink, i, opt->set.min);
150 if (opt->parent == TC_H_ROOT) {
152 err = __nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 1,
154 } else if (TC_H_MAJ(alink->parent) == TC_H_MAJ(opt->parent)) {
155 i = TC_H_MIN(opt->parent) - 1;
156 err = nfp_abm_ctrl_set_q_lvl(alink, i, opt->set.min);
160 /* Set the handle to try full clean up, in case IO failed */
161 alink->qdiscs[i].handle = opt->handle;
165 if (opt->parent == TC_H_ROOT)
166 err = nfp_abm_ctrl_read_stats(alink, &alink->qdiscs[i].stats);
168 err = nfp_abm_ctrl_read_q_stats(alink, i,
169 &alink->qdiscs[i].stats);
173 if (opt->parent == TC_H_ROOT)
174 err = nfp_abm_ctrl_read_xstats(alink,
175 &alink->qdiscs[i].xstats);
177 err = nfp_abm_ctrl_read_q_xstats(alink, i,
178 &alink->qdiscs[i].xstats);
182 alink->qdiscs[i].stats.backlog_pkts = 0;
183 alink->qdiscs[i].stats.backlog_bytes = 0;
187 /* If the qdisc keeps on living, but we can't offload undo changes */
189 opt->set.qstats->qlen -= alink->qdiscs[i].stats.backlog_pkts;
190 opt->set.qstats->backlog -=
191 alink->qdiscs[i].stats.backlog_bytes;
193 nfp_abm_red_destroy(netdev, alink, opt->handle);
199 nfp_abm_update_stats(struct nfp_alink_stats *new, struct nfp_alink_stats *old,
200 struct tc_qopt_offload_stats *stats)
202 _bstats_update(stats->bstats, new->tx_bytes - old->tx_bytes,
203 new->tx_pkts - old->tx_pkts);
204 stats->qstats->qlen += new->backlog_pkts - old->backlog_pkts;
205 stats->qstats->backlog += new->backlog_bytes - old->backlog_bytes;
206 stats->qstats->overlimits += new->overlimits - old->overlimits;
207 stats->qstats->drops += new->drops - old->drops;
211 nfp_abm_red_stats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt)
213 struct nfp_alink_stats *prev_stats;
214 struct nfp_alink_stats stats;
217 i = nfp_abm_red_find(alink, opt);
220 prev_stats = &alink->qdiscs[i].stats;
222 if (alink->parent == TC_H_ROOT)
223 err = nfp_abm_ctrl_read_stats(alink, &stats);
225 err = nfp_abm_ctrl_read_q_stats(alink, i, &stats);
229 nfp_abm_update_stats(&stats, prev_stats, &opt->stats);
237 nfp_abm_red_xstats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt)
239 struct nfp_alink_xstats *prev_xstats;
240 struct nfp_alink_xstats xstats;
243 i = nfp_abm_red_find(alink, opt);
246 prev_xstats = &alink->qdiscs[i].xstats;
248 if (alink->parent == TC_H_ROOT)
249 err = nfp_abm_ctrl_read_xstats(alink, &xstats);
251 err = nfp_abm_ctrl_read_q_xstats(alink, i, &xstats);
255 opt->xstats->forced_mark += xstats.ecn_marked - prev_xstats->ecn_marked;
256 opt->xstats->pdrop += xstats.pdrop - prev_xstats->pdrop;
258 *prev_xstats = xstats;
264 nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink,
265 struct tc_red_qopt_offload *opt)
267 switch (opt->command) {
269 return nfp_abm_red_replace(netdev, alink, opt);
271 nfp_abm_red_destroy(netdev, alink, opt->handle);
274 return nfp_abm_red_stats(alink, opt);
276 return nfp_abm_red_xstats(alink, opt);
283 nfp_abm_mq_stats(struct nfp_abm_link *alink, struct tc_mq_qopt_offload *opt)
285 struct nfp_alink_stats stats;
289 for (i = 0; i < alink->num_qdiscs; i++) {
290 if (alink->qdiscs[i].handle == TC_H_UNSPEC)
293 err = nfp_abm_ctrl_read_q_stats(alink, i, &stats);
297 nfp_abm_update_stats(&stats, &alink->qdiscs[i].stats,
305 nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink,
306 struct tc_mq_qopt_offload *opt)
308 switch (opt->command) {
310 nfp_abm_reset_root(netdev, alink, opt->handle,
311 alink->total_queues);
314 if (opt->handle == alink->parent)
315 nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 0);
318 return nfp_abm_mq_stats(alink, opt);
325 nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev,
326 enum tc_setup_type type, void *type_data)
328 struct nfp_repr *repr = netdev_priv(netdev);
329 struct nfp_port *port;
331 port = nfp_port_from_netdev(netdev);
332 if (!port || port->type != NFP_PORT_PF_PORT)
336 case TC_SETUP_QDISC_MQ:
337 return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data);
338 case TC_SETUP_QDISC_RED:
339 return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data);
345 static struct net_device *nfp_abm_repr_get(struct nfp_app *app, u32 port_id)
347 enum nfp_repr_type rtype;
348 struct nfp_reprs *reprs;
351 rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id);
352 port = FIELD_GET(NFP_ABM_PORTID_ID, port_id);
354 reprs = rcu_dereference(app->reprs[rtype]);
358 if (port >= reprs->num_reprs)
361 return rcu_dereference(reprs->reprs[port]);
365 nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink,
366 enum nfp_port_type ptype)
368 struct net_device *netdev;
369 enum nfp_repr_type rtype;
370 struct nfp_reprs *reprs;
371 struct nfp_repr *repr;
372 struct nfp_port *port;
376 if (ptype == NFP_PORT_PHYS_PORT) {
377 rtype = NFP_REPR_TYPE_PHYS_PORT;
380 rtype = NFP_REPR_TYPE_PF;
381 txqs = alink->vnic->max_rx_rings;
384 netdev = nfp_repr_alloc_mqs(app, txqs, 1);
387 repr = netdev_priv(netdev);
388 repr->app_priv = alink;
390 port = nfp_port_alloc(app, ptype, netdev);
396 if (ptype == NFP_PORT_PHYS_PORT) {
397 port->eth_forced = true;
398 err = nfp_port_init_phy_port(app->pf, app, port, alink->id);
402 port->pf_id = alink->abm->pf_id;
403 port->pf_split = app->pf->max_data_vnics > 1;
404 port->pf_split_id = alink->id;
405 port->vnic = alink->vnic->dp.ctrl_bar;
408 SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev);
409 eth_hw_addr_random(netdev);
411 err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id),
412 port, alink->vnic->dp.netdev);
416 reprs = nfp_reprs_get_locked(app, rtype);
417 WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr");
418 rcu_assign_pointer(reprs->reprs[alink->id], netdev);
420 nfp_info(app->cpp, "%s Port %d Representor(%s) created\n",
421 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys",
422 alink->id, netdev->name);
429 nfp_repr_free(netdev);
434 nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink,
435 enum nfp_repr_type rtype)
437 struct net_device *netdev;
438 struct nfp_reprs *reprs;
440 reprs = nfp_reprs_get_locked(app, rtype);
441 netdev = nfp_repr_get_locked(app, reprs, alink->id);
444 rcu_assign_pointer(reprs->reprs[alink->id], NULL);
446 /* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */
447 nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev));
451 nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink)
453 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF);
454 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT);
457 static void nfp_abm_kill_reprs_all(struct nfp_abm *abm)
459 struct nfp_pf *pf = abm->app->pf;
462 list_for_each_entry(nn, &pf->vnics, vnic_list)
463 nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv);
466 static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app)
468 struct nfp_abm *abm = app->priv;
470 return abm->eswitch_mode;
473 static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm)
475 nfp_abm_kill_reprs_all(abm);
476 nfp_abm_ctrl_qm_disable(abm);
478 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
482 static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm)
484 if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY)
485 WARN_ON(nfp_abm_eswitch_set_legacy(abm));
488 static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm)
490 struct nfp_app *app = abm->app;
491 struct nfp_pf *pf = app->pf;
495 err = nfp_abm_ctrl_qm_enable(abm);
499 list_for_each_entry(nn, &pf->vnics, vnic_list) {
500 struct nfp_abm_link *alink = nn->app_priv;
502 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT);
504 goto err_kill_all_reprs;
506 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT);
508 goto err_kill_all_reprs;
511 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
515 nfp_abm_kill_reprs_all(abm);
516 nfp_abm_ctrl_qm_disable(abm);
520 static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode)
522 struct nfp_abm *abm = app->priv;
524 if (abm->eswitch_mode == mode)
528 case DEVLINK_ESWITCH_MODE_LEGACY:
529 return nfp_abm_eswitch_set_legacy(abm);
530 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
531 return nfp_abm_eswitch_set_switchdev(abm);
538 nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn,
541 struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id];
542 u8 mac_addr[ETH_ALEN];
546 if (id > pf->eth_tbl->count) {
547 nfp_warn(pf->cpp, "No entry for persistent MAC address\n");
548 eth_hw_addr_random(nn->dp.netdev);
552 snprintf(name, sizeof(name), "eth%u.mac.pf%u",
553 eth_port->eth_index, abm->pf_id);
555 mac_str = nfp_hwinfo_lookup(pf->hwinfo, name);
557 nfp_warn(pf->cpp, "Can't lookup persistent MAC address (%s)\n",
559 eth_hw_addr_random(nn->dp.netdev);
563 if (sscanf(mac_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
564 &mac_addr[0], &mac_addr[1], &mac_addr[2],
565 &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) {
566 nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n",
568 eth_hw_addr_random(nn->dp.netdev);
572 ether_addr_copy(nn->dp.netdev->dev_addr, mac_addr);
573 ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr);
577 nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
579 struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id];
580 struct nfp_abm *abm = app->priv;
581 struct nfp_abm_link *alink;
584 alink = kzalloc(sizeof(*alink), GFP_KERNEL);
587 nn->app_priv = alink;
591 alink->parent = TC_H_ROOT;
592 alink->total_queues = alink->vnic->max_rx_rings;
593 alink->qdiscs = kvcalloc(alink->total_queues, sizeof(*alink->qdiscs),
595 if (!alink->qdiscs) {
600 /* This is a multi-host app, make sure MAC/PHY is up, but don't
601 * make the MAC/PHY state follow the state of any of the ports.
603 err = nfp_eth_set_configured(app->cpp, eth_port->index, true);
605 goto err_free_qdiscs;
607 netif_keep_dst(nn->dp.netdev);
609 nfp_abm_vnic_set_mac(app->pf, abm, nn, id);
610 nfp_abm_ctrl_read_params(alink);
615 kvfree(alink->qdiscs);
621 static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn)
623 struct nfp_abm_link *alink = nn->app_priv;
625 nfp_abm_kill_reprs(alink->abm, alink);
626 kvfree(alink->qdiscs);
631 nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data)
633 struct nfp_repr *repr = netdev_priv(port->netdev);
634 struct nfp_abm_link *alink;
637 if (port->type != NFP_PORT_PF_PORT)
639 alink = repr->app_priv;
640 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) {
641 *data++ = nfp_abm_ctrl_stat_non_sto(alink, i);
642 *data++ = nfp_abm_ctrl_stat_sto(alink, i);
648 nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port)
650 struct nfp_repr *repr = netdev_priv(port->netdev);
651 struct nfp_abm_link *alink;
653 if (port->type != NFP_PORT_PF_PORT)
655 alink = repr->app_priv;
656 return alink->vnic->dp.num_r_vecs * 2;
660 nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port,
663 struct nfp_repr *repr = netdev_priv(port->netdev);
664 struct nfp_abm_link *alink;
667 if (port->type != NFP_PORT_PF_PORT)
669 alink = repr->app_priv;
670 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) {
671 data = nfp_pr_et(data, "q%u_no_wait", i);
672 data = nfp_pr_et(data, "q%u_delayed", i);
677 static int nfp_abm_init(struct nfp_app *app)
679 struct nfp_pf *pf = app->pf;
680 struct nfp_reprs *reprs;
685 nfp_err(pf->cpp, "ABM NIC requires ETH table\n");
688 if (pf->max_data_vnics != pf->eth_tbl->count) {
689 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
690 pf->max_data_vnics, pf->eth_tbl->count);
693 if (!pf->mac_stats_bar) {
694 nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n");
698 abm = kzalloc(sizeof(*abm), GFP_KERNEL);
704 err = nfp_abm_ctrl_find_addrs(abm);
708 /* We start in legacy mode, make sure advanced queuing is disabled */
709 err = nfp_abm_ctrl_qm_disable(abm);
714 reprs = nfp_reprs_alloc(pf->max_data_vnics);
717 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs);
719 reprs = nfp_reprs_alloc(pf->max_data_vnics);
722 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs);
727 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
734 static void nfp_abm_clean(struct nfp_app *app)
736 struct nfp_abm *abm = app->priv;
738 nfp_abm_eswitch_clean_up(abm);
739 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);
740 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
745 const struct nfp_app_type app_abm = {
746 .id = NFP_APP_ACTIVE_BUFFER_MGMT_NIC,
749 .init = nfp_abm_init,
750 .clean = nfp_abm_clean,
752 .vnic_alloc = nfp_abm_vnic_alloc,
753 .vnic_free = nfp_abm_vnic_free,
755 .port_get_stats = nfp_abm_port_get_stats,
756 .port_get_stats_count = nfp_abm_port_get_stats_count,
757 .port_get_stats_strings = nfp_abm_port_get_stats_strings,
759 .setup_tc = nfp_abm_setup_tc,
761 .eswitch_mode_get = nfp_abm_eswitch_mode_get,
762 .eswitch_mode_set = nfp_abm_eswitch_mode_set,
764 .repr_get = nfp_abm_repr_get,