drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /* Copyright 2022 NXP
   3  */
   4 #include <linux/filter.h>
   5 #include <linux/compiler.h>
   6 #include <linux/bpf_trace.h>
   7 #include <net/xdp.h>
   8 #include <net/xdp_sock_drv.h>
   9
  10 #include "dpaa2-eth.h"
  11
  12 static void dpaa2_eth_setup_consume_func(struct dpaa2_eth_priv *priv,
  13                                          struct dpaa2_eth_channel *ch,
  14                                          enum dpaa2_eth_fq_type type,
  15                                          dpaa2_eth_consume_cb_t *consume)
  16 {
  17         struct dpaa2_eth_fq *fq;
  18         int i;
  19
  20         for (i = 0; i < priv->num_fqs; i++) {
  21                 fq = &priv->fq[i];
  22
  23                 if (fq->type != type)
  24                         continue;
  25                 if (fq->channel != ch)
  26                         continue;
  27
  28                 fq->consume = consume;
  29         }
  30 }
  31
  32 static u32 dpaa2_xsk_run_xdp(struct dpaa2_eth_priv *priv,
  33                              struct dpaa2_eth_channel *ch,
  34                              struct dpaa2_eth_fq *rx_fq,
  35                              struct dpaa2_fd *fd, void *vaddr)
  36 {
  37         dma_addr_t addr = dpaa2_fd_get_addr(fd);
  38         struct bpf_prog *xdp_prog;
  39         struct xdp_buff *xdp_buff;
  40         struct dpaa2_eth_swa *swa;
  41         u32 xdp_act = XDP_PASS;
  42         int err;
  43
  44         xdp_prog = READ_ONCE(ch->xdp.prog);
  45         if (!xdp_prog)
  46                 goto out;
  47
  48         swa = (struct dpaa2_eth_swa *)(vaddr + DPAA2_ETH_RX_HWA_SIZE +
  49                                        ch->xsk_pool->umem->headroom);
  50         xdp_buff = swa->xsk.xdp_buff;
  51
  52         xdp_buff->data_hard_start = vaddr;
  53         xdp_buff->data = vaddr + dpaa2_fd_get_offset(fd);
  54         xdp_buff->data_end = xdp_buff->data + dpaa2_fd_get_len(fd);
  55         xdp_set_data_meta_invalid(xdp_buff);
  56         xdp_buff->rxq = &ch->xdp_rxq;
  57
  58         xsk_buff_dma_sync_for_cpu(xdp_buff, ch->xsk_pool);
  59         xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff);
  60
  61         /* xdp.data pointer may have changed */
  62         dpaa2_fd_set_offset(fd, xdp_buff->data - vaddr);
  63         dpaa2_fd_set_len(fd, xdp_buff->data_end - xdp_buff->data);
  64
  65         if (likely(xdp_act == XDP_REDIRECT)) {
  66                 err = xdp_do_redirect(priv->net_dev, xdp_buff, xdp_prog);
  67                 if (unlikely(err)) {
  68                         ch->stats.xdp_drop++;
  69                         dpaa2_eth_recycle_buf(priv, ch, addr);
  70                 } else {
  71                         ch->buf_count--;
  72                         ch->stats.xdp_redirect++;
  73                 }
  74
  75                 goto xdp_redir;
  76         }
  77
  78         switch (xdp_act) {
  79         case XDP_PASS:
  80                 break;
  81         case XDP_TX:
  82                 dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid);
  83                 break;
  84         default:
  85                 bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act);
  86                 fallthrough;
  87         case XDP_ABORTED:
  88                 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
  89                 fallthrough;
  90         case XDP_DROP:
  91                 dpaa2_eth_recycle_buf(priv, ch, addr);
  92                 ch->stats.xdp_drop++;
  93                 break;
  94         }
  95
  96 xdp_redir:
  97         ch->xdp.res |= xdp_act;
  98 out:
  99         return xdp_act;
 100 }
 101
 102 /* Rx frame processing routine for the AF_XDP fast path */
 103 static void dpaa2_xsk_rx(struct dpaa2_eth_priv *priv,
 104                          struct dpaa2_eth_channel *ch,
 105                          const struct dpaa2_fd *fd,
 106                          struct dpaa2_eth_fq *fq)
 107 {
 108         dma_addr_t addr = dpaa2_fd_get_addr(fd);
 109         u8 fd_format = dpaa2_fd_get_format(fd);
 110         struct rtnl_link_stats64 *percpu_stats;
 111         u32 fd_length = dpaa2_fd_get_len(fd);
 112         struct sk_buff *skb;
 113         void *vaddr;
 114         u32 xdp_act;
 115
 116         trace_dpaa2_rx_xsk_fd(priv->net_dev, fd);
 117
 118         vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
 119         percpu_stats = this_cpu_ptr(priv->percpu_stats);
 120
 121         if (fd_format != dpaa2_fd_single) {
 122                 WARN_ON(priv->xdp_prog);
 123                 /* AF_XDP doesn't support any other formats */
 124                 goto err_frame_format;
 125         }
 126
 127         xdp_act = dpaa2_xsk_run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr);
 128         if (xdp_act != XDP_PASS) {
 129                 percpu_stats->rx_packets++;
 130                 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
 131                 return;
 132         }
 133
 134         /* Build skb */
 135         skb = dpaa2_eth_alloc_skb(priv, ch, fd, fd_length, vaddr);
 136         if (!skb)
 137                 /* Nothing else we can do, recycle the buffer and
 138                  * drop the frame.
 139                  */
 140                 goto err_alloc_skb;
 141
 142         /* Send the skb to the Linux networking stack */
 143         dpaa2_eth_receive_skb(priv, ch, fd, vaddr, fq, percpu_stats, skb);
 144
 145         return;
 146
 147 err_alloc_skb:
 148         dpaa2_eth_recycle_buf(priv, ch, addr);
 149 err_frame_format:
 150         percpu_stats->rx_dropped++;
 151 }
 152
 153 static void dpaa2_xsk_set_bp_per_qdbin(struct dpaa2_eth_priv *priv,
 154                                        struct dpni_pools_cfg *pools_params)
 155 {
 156         int curr_bp = 0, i, j;
 157
 158         pools_params->pool_options = DPNI_POOL_ASSOC_QDBIN;
 159         for (i = 0; i < priv->num_bps; i++) {
 160                 for (j = 0; j < priv->num_channels; j++)
 161                         if (priv->bp[i] == priv->channel[j]->bp)
 162                                 pools_params->pools[curr_bp].priority_mask |= (1 << j);
 163                 if (!pools_params->pools[curr_bp].priority_mask)
 164                         continue;
 165
 166                 pools_params->pools[curr_bp].dpbp_id = priv->bp[i]->bpid;
 167                 pools_params->pools[curr_bp].buffer_size = priv->rx_buf_size;
 168                 pools_params->pools[curr_bp++].backup_pool = 0;
 169         }
 170         pools_params->num_dpbp = curr_bp;
 171 }
 172
 173 static int dpaa2_xsk_disable_pool(struct net_device *dev, u16 qid)
 174 {
 175         struct xsk_buff_pool *pool = xsk_get_pool_from_qid(dev, qid);
 176         struct dpaa2_eth_priv *priv = netdev_priv(dev);
 177         struct dpni_pools_cfg pools_params = { 0 };
 178         struct dpaa2_eth_channel *ch;
 179         int err;
 180         bool up;
 181
 182         ch = priv->channel[qid];
 183         if (!ch->xsk_pool)
 184                 return -EINVAL;
 185
 186         up = netif_running(dev);
 187         if (up)
 188                 dev_close(dev);
 189
 190         xsk_pool_dma_unmap(pool, 0);
 191         err = xdp_rxq_info_reg_mem_model(&ch->xdp_rxq,
 192                                          MEM_TYPE_PAGE_ORDER0, NULL);
 193         if (err)
 194                 netdev_err(dev, "xsk_rxq_info_reg_mem_model() failed (err = %d)\n",
 195                            err);
 196
 197         dpaa2_eth_free_dpbp(priv, ch->bp);
 198
 199         ch->xsk_zc = false;
 200         ch->xsk_pool = NULL;
 201         ch->xsk_tx_pkts_sent = 0;
 202         ch->bp = priv->bp[DPAA2_ETH_DEFAULT_BP_IDX];
 203
 204         dpaa2_eth_setup_consume_func(priv, ch, DPAA2_RX_FQ, dpaa2_eth_rx);
 205
 206         dpaa2_xsk_set_bp_per_qdbin(priv, &pools_params);
 207         err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);
 208         if (err)
 209                 netdev_err(dev, "dpni_set_pools() failed\n");
 210
 211         if (up) {
 212                 err = dev_open(dev, NULL);
 213                 if (err)
 214                         return err;
 215         }
 216
 217         return 0;
 218 }
 219
 220 static int dpaa2_xsk_enable_pool(struct net_device *dev,
 221                                  struct xsk_buff_pool *pool,
 222                                  u16 qid)
 223 {
 224         struct dpaa2_eth_priv *priv = netdev_priv(dev);
 225         struct dpni_pools_cfg pools_params = { 0 };
 226         struct dpaa2_eth_channel *ch;
 227         int err, err2;
 228         bool up;
 229
 230         if (priv->dpni_attrs.wriop_version < DPAA2_WRIOP_VERSION(3, 0, 0)) {
 231                 netdev_err(dev, "AF_XDP zero-copy not supported on devices <= WRIOP(3, 0, 0)\n");
 232                 return -EOPNOTSUPP;
 233         }
 234
 235         if (priv->dpni_attrs.num_queues > 8) {
 236                 netdev_err(dev, "AF_XDP zero-copy not supported on DPNI with more then 8 queues\n");
 237                 return -EOPNOTSUPP;
 238         }
 239
 240         up = netif_running(dev);
 241         if (up)
 242                 dev_close(dev);
 243
 244         err = xsk_pool_dma_map(pool, priv->net_dev->dev.parent, 0);
 245         if (err) {
 246                 netdev_err(dev, "xsk_pool_dma_map() failed (err = %d)\n",
 247                            err);
 248                 goto err_dma_unmap;
 249         }
 250
 251         ch = priv->channel[qid];
 252         err = xdp_rxq_info_reg_mem_model(&ch->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL);
 253         if (err) {
 254                 netdev_err(dev, "xdp_rxq_info_reg_mem_model() failed (err = %d)\n", err);
 255                 goto err_mem_model;
 256         }
 257         xsk_pool_set_rxq_info(pool, &ch->xdp_rxq);
 258
 259         priv->bp[priv->num_bps] = dpaa2_eth_allocate_dpbp(priv);
 260         if (IS_ERR(priv->bp[priv->num_bps])) {
 261                 err = PTR_ERR(priv->bp[priv->num_bps]);
 262                 goto err_bp_alloc;
 263         }
 264         ch->xsk_zc = true;
 265         ch->xsk_pool = pool;
 266         ch->bp = priv->bp[priv->num_bps++];
 267
 268         dpaa2_eth_setup_consume_func(priv, ch, DPAA2_RX_FQ, dpaa2_xsk_rx);
 269
 270         dpaa2_xsk_set_bp_per_qdbin(priv, &pools_params);
 271         err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);
 272         if (err) {
 273                 netdev_err(dev, "dpni_set_pools() failed\n");
 274                 goto err_set_pools;
 275         }
 276
 277         if (up) {
 278                 err = dev_open(dev, NULL);
 279                 if (err)
 280                         return err;
 281         }
 282
 283         return 0;
 284
 285 err_set_pools:
 286         err2 = dpaa2_xsk_disable_pool(dev, qid);
 287         if (err2)
 288                 netdev_err(dev, "dpaa2_xsk_disable_pool() failed %d\n", err2);
 289 err_bp_alloc:
 290         err2 = xdp_rxq_info_reg_mem_model(&priv->channel[qid]->xdp_rxq,
 291                                           MEM_TYPE_PAGE_ORDER0, NULL);
 292         if (err2)
 293                 netdev_err(dev, "xsk_rxq_info_reg_mem_model() failed with %d)\n", err2);
 294 err_mem_model:
 295         xsk_pool_dma_unmap(pool, 0);
 296 err_dma_unmap:
 297         if (up)
 298                 dev_open(dev, NULL);
 299
 300         return err;
 301 }
 302
 303 int dpaa2_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid)
 304 {
 305         return pool ? dpaa2_xsk_enable_pool(dev, pool, qid) :
 306                       dpaa2_xsk_disable_pool(dev, qid);
 307 }
 308
 309 int dpaa2_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
 310 {
 311         struct dpaa2_eth_priv *priv = netdev_priv(dev);
 312         struct dpaa2_eth_channel *ch = priv->channel[qid];
 313
 314         if (!priv->link_state.up)
 315                 return -ENETDOWN;
 316
 317         if (!priv->xdp_prog)
 318                 return -EINVAL;
 319
 320         if (!ch->xsk_zc)
 321                 return -EINVAL;
 322
 323         /* We do not have access to a per channel SW interrupt, so instead we
 324          * schedule a NAPI instance.
 325          */
 326         if (!napi_if_scheduled_mark_missed(&ch->napi))
 327                 napi_schedule(&ch->napi);
 328
 329         return 0;
 330 }
 331
 332 static int dpaa2_xsk_tx_build_fd(struct dpaa2_eth_priv *priv,
 333                                  struct dpaa2_eth_channel *ch,
 334                                  struct dpaa2_fd *fd,
 335                                  struct xdp_desc *xdp_desc)
 336 {
 337         struct device *dev = priv->net_dev->dev.parent;
 338         struct dpaa2_sg_entry *sgt;
 339         struct dpaa2_eth_swa *swa;
 340         void *sgt_buf = NULL;
 341         dma_addr_t sgt_addr;
 342         int sgt_buf_size;
 343         dma_addr_t addr;
 344         int err = 0;
 345
 346         /* Prepare the HW SGT structure */
 347         sgt_buf_size = priv->tx_data_offset + sizeof(struct dpaa2_sg_entry);
 348         sgt_buf = dpaa2_eth_sgt_get(priv);
 349         if (unlikely(!sgt_buf))
 350                 return -ENOMEM;
 351         sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset);
 352
 353         /* Get the address of the XSK Tx buffer */
 354         addr = xsk_buff_raw_get_dma(ch->xsk_pool, xdp_desc->addr);
 355         xsk_buff_raw_dma_sync_for_device(ch->xsk_pool, addr, xdp_desc->len);
 356
 357         /* Fill in the HW SGT structure */
 358         dpaa2_sg_set_addr(sgt, addr);
 359         dpaa2_sg_set_len(sgt, xdp_desc->len);
 360         dpaa2_sg_set_final(sgt, true);
 361
 362         /* Store the necessary info in the SGT buffer */
 363         swa = (struct dpaa2_eth_swa *)sgt_buf;
 364         swa->type = DPAA2_ETH_SWA_XSK;
 365         swa->xsk.sgt_size = sgt_buf_size;
 366
 367         /* Separately map the SGT buffer */
 368         sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL);
 369         if (unlikely(dma_mapping_error(dev, sgt_addr))) {
 370                 err = -ENOMEM;
 371                 goto sgt_map_failed;
 372         }
 373
 374         /* Initialize FD fields */
 375         memset(fd, 0, sizeof(struct dpaa2_fd));
 376         dpaa2_fd_set_offset(fd, priv->tx_data_offset);
 377         dpaa2_fd_set_format(fd, dpaa2_fd_sg);
 378         dpaa2_fd_set_addr(fd, sgt_addr);
 379         dpaa2_fd_set_len(fd, xdp_desc->len);
 380         dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
 381
 382         return 0;
 383
 384 sgt_map_failed:
 385         dpaa2_eth_sgt_recycle(priv, sgt_buf);
 386
 387         return err;
 388 }
 389
 390 bool dpaa2_xsk_tx(struct dpaa2_eth_priv *priv,
 391                   struct dpaa2_eth_channel *ch)
 392 {
 393         struct xdp_desc *xdp_descs = ch->xsk_pool->tx_descs;
 394         struct dpaa2_eth_drv_stats *percpu_extras;
 395         struct rtnl_link_stats64 *percpu_stats;
 396         int budget = DPAA2_ETH_TX_ZC_PER_NAPI;
 397         int total_enqueued, enqueued;
 398         int retries, max_retries;
 399         struct dpaa2_eth_fq *fq;
 400         struct dpaa2_fd *fds;
 401         int batch, i, err;
 402
 403         percpu_stats = this_cpu_ptr(priv->percpu_stats);
 404         percpu_extras = this_cpu_ptr(priv->percpu_extras);
 405         fds = (this_cpu_ptr(priv->fd))->array;
 406
 407         /* Use the FQ with the same idx as the affine CPU */
 408         fq = &priv->fq[ch->nctx.desired_cpu];
 409
 410         batch = xsk_tx_peek_release_desc_batch(ch->xsk_pool, budget);
 411         if (!batch)
 412                 return false;
 413
 414         /* Create a FD for each XSK frame to be sent */
 415         for (i = 0; i < batch; i++) {
 416                 err = dpaa2_xsk_tx_build_fd(priv, ch, &fds[i], &xdp_descs[i]);
 417                 if (err) {
 418                         batch = i;
 419                         break;
 420                 }
 421
 422                 trace_dpaa2_tx_xsk_fd(priv->net_dev, &fds[i]);
 423         }
 424
 425         /* Enqueue all the created FDs */
 426         max_retries = batch * DPAA2_ETH_ENQUEUE_RETRIES;
 427         total_enqueued = 0;
 428         enqueued = 0;
 429         retries = 0;
 430         while (total_enqueued < batch && retries < max_retries) {
 431                 err = priv->enqueue(priv, fq, &fds[total_enqueued], 0,
 432                                     batch - total_enqueued, &enqueued);
 433                 if (err == -EBUSY) {
 434                         retries++;
 435                         continue;
 436                 }
 437
 438                 total_enqueued += enqueued;
 439         }
 440         percpu_extras->tx_portal_busy += retries;
 441
 442         /* Update statistics */
 443         percpu_stats->tx_packets += total_enqueued;
 444         for (i = 0; i < total_enqueued; i++)
 445                 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]);
 446         for (i = total_enqueued; i < batch; i++) {
 447                 dpaa2_eth_free_tx_fd(priv, ch, fq, &fds[i], false);
 448                 percpu_stats->tx_errors++;
 449         }
 450
 451         xsk_tx_release(ch->xsk_pool);
 452
 453         return total_enqueued == budget;
 454 }