drivers/dma/ti/k3-udma-glue.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * K3 NAVSS DMA glue interface
   4  *
   5  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  */
   8
   9 #include <linux/atomic.h>
  10 #include <linux/delay.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/io.h>
  13 #include <linux/init.h>
  14 #include <linux/of.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/soc/ti/k3-ringacc.h>
  17 #include <linux/dma/ti-cppi5.h>
  18 #include <linux/dma/k3-udma-glue.h>
  19
  20 #include "k3-udma.h"
  21 #include "k3-psil-priv.h"
  22
  23 struct k3_udma_glue_common {
  24         struct device *dev;
  25         struct device chan_dev;
  26         struct udma_dev *udmax;
  27         const struct udma_tisci_rm *tisci_rm;
  28         struct k3_ringacc *ringacc;
  29         u32 src_thread;
  30         u32 dst_thread;
  31
  32         u32  hdesc_size;
  33         bool epib;
  34         u32  psdata_size;
  35         u32  swdata_size;
  36         u32  atype_asel;
  37         struct psil_endpoint_config *ep_config;
  38 };
  39
  40 struct k3_udma_glue_tx_channel {
  41         struct k3_udma_glue_common common;
  42
  43         struct udma_tchan *udma_tchanx;
  44         int udma_tchan_id;
  45
  46         struct k3_ring *ringtx;
  47         struct k3_ring *ringtxcq;
  48
  49         bool psil_paired;
  50
  51         int virq;
  52
  53         atomic_t free_pkts;
  54         bool tx_pause_on_err;
  55         bool tx_filt_einfo;
  56         bool tx_filt_pswords;
  57         bool tx_supr_tdpkt;
  58
  59         int udma_tflow_id;
  60 };
  61
  62 struct k3_udma_glue_rx_flow {
  63         struct udma_rflow *udma_rflow;
  64         int udma_rflow_id;
  65         struct k3_ring *ringrx;
  66         struct k3_ring *ringrxfdq;
  67
  68         int virq;
  69 };
  70
  71 struct k3_udma_glue_rx_channel {
  72         struct k3_udma_glue_common common;
  73
  74         struct udma_rchan *udma_rchanx;
  75         int udma_rchan_id;
  76         bool remote;
  77
  78         bool psil_paired;
  79
  80         u32  swdata_size;
  81         int  flow_id_base;
  82
  83         struct k3_udma_glue_rx_flow *flows;
  84         u32 flow_num;
  85         u32 flows_ready;
  86 };
  87
  88 static void k3_udma_chan_dev_release(struct device *dev)
  89 {
  90         /* The struct containing the device is devm managed */
  91 }
  92
  93 static struct class k3_udma_glue_devclass = {
  94         .name           = "k3_udma_glue_chan",
  95         .dev_release    = k3_udma_chan_dev_release,
  96 };
  97
  98 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000
  99
 100 static int of_k3_udma_glue_parse(struct device_node *udmax_np,
 101                                  struct k3_udma_glue_common *common)
 102 {
 103         common->udmax = of_xudma_dev_get(udmax_np, NULL);
 104         if (IS_ERR(common->udmax))
 105                 return PTR_ERR(common->udmax);
 106
 107         common->ringacc = xudma_get_ringacc(common->udmax);
 108         common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
 109
 110         return 0;
 111 }
 112
 113 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
 114                 const char *name, struct k3_udma_glue_common *common,
 115                 bool tx_chn)
 116 {
 117         struct of_phandle_args dma_spec;
 118         u32 thread_id;
 119         int ret = 0;
 120         int index;
 121
 122         if (unlikely(!name))
 123                 return -EINVAL;
 124
 125         index = of_property_match_string(chn_np, "dma-names", name);
 126         if (index < 0)
 127                 return index;
 128
 129         if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index,
 130                                        &dma_spec))
 131                 return -ENOENT;
 132
 133         ret = of_k3_udma_glue_parse(dma_spec.np, common);
 134         if (ret)
 135                 goto out_put_spec;
 136
 137         thread_id = dma_spec.args[0];
 138         if (dma_spec.args_count == 2) {
 139                 if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) {
 140                         dev_err(common->dev, "Invalid channel atype: %u\n",
 141                                 dma_spec.args[1]);
 142                         ret = -EINVAL;
 143                         goto out_put_spec;
 144                 }
 145                 if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) {
 146                         dev_err(common->dev, "Invalid channel asel: %u\n",
 147                                 dma_spec.args[1]);
 148                         ret = -EINVAL;
 149                         goto out_put_spec;
 150                 }
 151
 152                 common->atype_asel = dma_spec.args[1];
 153         }
 154
 155         if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
 156                 ret = -EINVAL;
 157                 goto out_put_spec;
 158         }
 159
 160         if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
 161                 ret = -EINVAL;
 162                 goto out_put_spec;
 163         }
 164
 165         /* get psil endpoint config */
 166         common->ep_config = psil_get_ep_config(thread_id);
 167         if (IS_ERR(common->ep_config)) {
 168                 dev_err(common->dev,
 169                         "No configuration for psi-l thread 0x%04x\n",
 170                         thread_id);
 171                 ret = PTR_ERR(common->ep_config);
 172                 goto out_put_spec;
 173         }
 174
 175         common->epib = common->ep_config->needs_epib;
 176         common->psdata_size = common->ep_config->psd_size;
 177
 178         if (tx_chn)
 179                 common->dst_thread = thread_id;
 180         else
 181                 common->src_thread = thread_id;
 182
 183 out_put_spec:
 184         of_node_put(dma_spec.np);
 185         return ret;
 186 };
 187
 188 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 189 {
 190         struct device *dev = tx_chn->common.dev;
 191
 192         dev_dbg(dev, "dump_tx_chn:\n"
 193                 "udma_tchan_id: %d\n"
 194                 "src_thread: %08x\n"
 195                 "dst_thread: %08x\n",
 196                 tx_chn->udma_tchan_id,
 197                 tx_chn->common.src_thread,
 198                 tx_chn->common.dst_thread);
 199 }
 200
 201 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn,
 202                                         char *mark)
 203 {
 204         struct device *dev = chn->common.dev;
 205
 206         dev_dbg(dev, "=== dump ===> %s\n", mark);
 207         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
 208                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG));
 209         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
 210                 xudma_tchanrt_read(chn->udma_tchanx,
 211                                    UDMA_CHAN_RT_PEER_RT_EN_REG));
 212         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
 213                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_PCNT_REG));
 214         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
 215                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_BCNT_REG));
 216         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
 217                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_SBCNT_REG));
 218 }
 219
 220 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 221 {
 222         const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm;
 223         struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
 224
 225         memset(&req, 0, sizeof(req));
 226
 227         req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
 228                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
 229                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
 230                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
 231                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
 232                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
 233                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
 234                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
 235         req.nav_id = tisci_rm->tisci_dev_id;
 236         req.index = tx_chn->udma_tchan_id;
 237         if (tx_chn->tx_pause_on_err)
 238                 req.tx_pause_on_err = 1;
 239         if (tx_chn->tx_filt_einfo)
 240                 req.tx_filt_einfo = 1;
 241         if (tx_chn->tx_filt_pswords)
 242                 req.tx_filt_pswords = 1;
 243         req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
 244         if (tx_chn->tx_supr_tdpkt)
 245                 req.tx_supr_tdpkt = 1;
 246         req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
 247         req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 248         req.tx_atype = tx_chn->common.atype_asel;
 249
 250         return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
 251 }
 252
 253 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
 254                 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg)
 255 {
 256         struct k3_udma_glue_tx_channel *tx_chn;
 257         int ret;
 258
 259         tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL);
 260         if (!tx_chn)
 261                 return ERR_PTR(-ENOMEM);
 262
 263         tx_chn->common.dev = dev;
 264         tx_chn->common.swdata_size = cfg->swdata_size;
 265         tx_chn->tx_pause_on_err = cfg->tx_pause_on_err;
 266         tx_chn->tx_filt_einfo = cfg->tx_filt_einfo;
 267         tx_chn->tx_filt_pswords = cfg->tx_filt_pswords;
 268         tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt;
 269
 270         /* parse of udmap channel */
 271         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
 272                                         &tx_chn->common, true);
 273         if (ret)
 274                 goto err;
 275
 276         tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib,
 277                                                 tx_chn->common.psdata_size,
 278                                                 tx_chn->common.swdata_size);
 279
 280         if (xudma_is_pktdma(tx_chn->common.udmax))
 281                 tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id;
 282         else
 283                 tx_chn->udma_tchan_id = -1;
 284
 285         /* request and cfg UDMAP TX channel */
 286         tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax,
 287                                               tx_chn->udma_tchan_id);
 288         if (IS_ERR(tx_chn->udma_tchanx)) {
 289                 ret = PTR_ERR(tx_chn->udma_tchanx);
 290                 dev_err(dev, "UDMAX tchanx get err %d\n", ret);
 291                 goto err;
 292         }
 293         tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
 294
 295         tx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
 296         tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax);
 297         dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x",
 298                      tx_chn->udma_tchan_id, tx_chn->common.dst_thread);
 299         ret = device_register(&tx_chn->common.chan_dev);
 300         if (ret) {
 301                 dev_err(dev, "Channel Device registration failed %d\n", ret);
 302                 tx_chn->common.chan_dev.parent = NULL;
 303                 goto err;
 304         }
 305
 306         if (xudma_is_pktdma(tx_chn->common.udmax)) {
 307                 /* prepare the channel device as coherent */
 308                 tx_chn->common.chan_dev.dma_coherent = true;
 309                 dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
 310                                              DMA_BIT_MASK(48));
 311         }
 312
 313         atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
 314
 315         if (xudma_is_pktdma(tx_chn->common.udmax))
 316                 tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id;
 317         else
 318                 tx_chn->udma_tflow_id = tx_chn->udma_tchan_id;
 319
 320         /* request and cfg rings */
 321         ret =  k3_ringacc_request_rings_pair(tx_chn->common.ringacc,
 322                                              tx_chn->udma_tflow_id, -1,
 323                                              &tx_chn->ringtx,
 324                                              &tx_chn->ringtxcq);
 325         if (ret) {
 326                 dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret);
 327                 goto err;
 328         }
 329
 330         /* Set the dma_dev for the rings to be configured */
 331         cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn);
 332         cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev;
 333
 334         /* Set the ASEL value for DMA rings of PKTDMA */
 335         if (xudma_is_pktdma(tx_chn->common.udmax)) {
 336                 cfg->tx_cfg.asel = tx_chn->common.atype_asel;
 337                 cfg->txcq_cfg.asel = tx_chn->common.atype_asel;
 338         }
 339
 340         ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
 341         if (ret) {
 342                 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
 343                 goto err;
 344         }
 345
 346         ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg);
 347         if (ret) {
 348                 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
 349                 goto err;
 350         }
 351
 352         /* request and cfg psi-l */
 353         tx_chn->common.src_thread =
 354                         xudma_dev_get_psil_base(tx_chn->common.udmax) +
 355                         tx_chn->udma_tchan_id;
 356
 357         ret = k3_udma_glue_cfg_tx_chn(tx_chn);
 358         if (ret) {
 359                 dev_err(dev, "Failed to cfg tchan %d\n", ret);
 360                 goto err;
 361         }
 362
 363         k3_udma_glue_dump_tx_chn(tx_chn);
 364
 365         return tx_chn;
 366
 367 err:
 368         k3_udma_glue_release_tx_chn(tx_chn);
 369         return ERR_PTR(ret);
 370 }
 371 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn);
 372
 373 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 374 {
 375         if (tx_chn->psil_paired) {
 376                 xudma_navss_psil_unpair(tx_chn->common.udmax,
 377                                         tx_chn->common.src_thread,
 378                                         tx_chn->common.dst_thread);
 379                 tx_chn->psil_paired = false;
 380         }
 381
 382         if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx))
 383                 xudma_tchan_put(tx_chn->common.udmax,
 384                                 tx_chn->udma_tchanx);
 385
 386         if (tx_chn->ringtxcq)
 387                 k3_ringacc_ring_free(tx_chn->ringtxcq);
 388
 389         if (tx_chn->ringtx)
 390                 k3_ringacc_ring_free(tx_chn->ringtx);
 391
 392         if (tx_chn->common.chan_dev.parent) {
 393                 device_unregister(&tx_chn->common.chan_dev);
 394                 tx_chn->common.chan_dev.parent = NULL;
 395         }
 396 }
 397 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn);
 398
 399 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 400                              struct cppi5_host_desc_t *desc_tx,
 401                              dma_addr_t desc_dma)
 402 {
 403         u32 ringtxcq_id;
 404
 405         if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0))
 406                 return -ENOMEM;
 407
 408         ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 409         cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id);
 410
 411         return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma);
 412 }
 413 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn);
 414
 415 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 416                             dma_addr_t *desc_dma)
 417 {
 418         int ret;
 419
 420         ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma);
 421         if (!ret)
 422                 atomic_inc(&tx_chn->free_pkts);
 423
 424         return ret;
 425 }
 426 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn);
 427
 428 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 429 {
 430         int ret;
 431
 432         ret = xudma_navss_psil_pair(tx_chn->common.udmax,
 433                                     tx_chn->common.src_thread,
 434                                     tx_chn->common.dst_thread);
 435         if (ret) {
 436                 dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret);
 437                 return ret;
 438         }
 439
 440         tx_chn->psil_paired = true;
 441
 442         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
 443                             UDMA_PEER_RT_EN_ENABLE);
 444
 445         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
 446                             UDMA_CHAN_RT_CTL_EN);
 447
 448         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en");
 449         return 0;
 450 }
 451 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn);
 452
 453 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 454 {
 455         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1");
 456
 457         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 0);
 458
 459         xudma_tchanrt_write(tx_chn->udma_tchanx,
 460                             UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
 461         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2");
 462
 463         if (tx_chn->psil_paired) {
 464                 xudma_navss_psil_unpair(tx_chn->common.udmax,
 465                                         tx_chn->common.src_thread,
 466                                         tx_chn->common.dst_thread);
 467                 tx_chn->psil_paired = false;
 468         }
 469 }
 470 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn);
 471
 472 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 473                                bool sync)
 474 {
 475         int i = 0;
 476         u32 val;
 477
 478         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1");
 479
 480         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
 481                             UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN);
 482
 483         val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG);
 484
 485         while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
 486                 val = xudma_tchanrt_read(tx_chn->udma_tchanx,
 487                                          UDMA_CHAN_RT_CTL_REG);
 488                 udelay(1);
 489                 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
 490                         dev_err(tx_chn->common.dev, "TX tdown timeout\n");
 491                         break;
 492                 }
 493                 i++;
 494         }
 495
 496         val = xudma_tchanrt_read(tx_chn->udma_tchanx,
 497                                  UDMA_CHAN_RT_PEER_RT_EN_REG);
 498         if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
 499                 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n");
 500         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2");
 501 }
 502 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn);
 503
 504 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 505                                void *data,
 506                                void (*cleanup)(void *data, dma_addr_t desc_dma))
 507 {
 508         struct device *dev = tx_chn->common.dev;
 509         dma_addr_t desc_dma;
 510         int occ_tx, i, ret;
 511
 512         /*
 513          * TXQ reset need to be special way as it is input for udma and its
 514          * state cached by udma, so:
 515          * 1) save TXQ occ
 516          * 2) clean up TXQ and call callback .cleanup() for each desc
 517          * 3) reset TXQ in a special way
 518          */
 519         occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
 520         dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx);
 521
 522         for (i = 0; i < occ_tx; i++) {
 523                 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
 524                 if (ret) {
 525                         if (ret != -ENODATA)
 526                                 dev_err(dev, "TX reset pop %d\n", ret);
 527                         break;
 528                 }
 529                 cleanup(data, desc_dma);
 530         }
 531
 532         /* reset TXCQ as it is not input for udma - expected to be empty */
 533         k3_ringacc_ring_reset(tx_chn->ringtxcq);
 534         k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
 535 }
 536 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn);
 537
 538 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn)
 539 {
 540         return tx_chn->common.hdesc_size;
 541 }
 542 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size);
 543
 544 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn)
 545 {
 546         return k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 547 }
 548 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id);
 549
 550 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn)
 551 {
 552         if (xudma_is_pktdma(tx_chn->common.udmax)) {
 553                 tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax,
 554                                                           tx_chn->udma_tflow_id);
 555         } else {
 556                 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
 557         }
 558
 559         return tx_chn->virq;
 560 }
 561 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq);
 562
 563 struct device *
 564         k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn)
 565 {
 566         if (xudma_is_pktdma(tx_chn->common.udmax) &&
 567             (tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15))
 568                 return &tx_chn->common.chan_dev;
 569
 570         return xudma_get_device(tx_chn->common.udmax);
 571 }
 572 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device);
 573
 574 void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn,
 575                                        dma_addr_t *addr)
 576 {
 577         if (!xudma_is_pktdma(tx_chn->common.udmax) ||
 578             !tx_chn->common.atype_asel)
 579                 return;
 580
 581         *addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
 582 }
 583 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr);
 584
 585 void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn,
 586                                        dma_addr_t *addr)
 587 {
 588         if (!xudma_is_pktdma(tx_chn->common.udmax) ||
 589             !tx_chn->common.atype_asel)
 590                 return;
 591
 592         *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
 593 }
 594 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr);
 595
 596 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
 597 {
 598         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
 599         struct ti_sci_msg_rm_udmap_rx_ch_cfg req;
 600         int ret;
 601
 602         memset(&req, 0, sizeof(req));
 603
 604         req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
 605                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
 606                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
 607                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
 608
 609         req.nav_id = tisci_rm->tisci_dev_id;
 610         req.index = rx_chn->udma_rchan_id;
 611         req.rx_fetch_size = rx_chn->common.hdesc_size >> 2;
 612         /*
 613          * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw
 614          * and udmax impl, so just configure it to invalid value.
 615          * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
 616          */
 617         req.rxcq_qnum = 0xFFFF;
 618         if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num &&
 619             rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
 620                 /* Default flow + extra ones */
 621                 req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
 622                                     TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
 623                 req.flowid_start = rx_chn->flow_id_base;
 624                 req.flowid_cnt = rx_chn->flow_num;
 625         }
 626         req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
 627         req.rx_atype = rx_chn->common.atype_asel;
 628
 629         ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
 630         if (ret)
 631                 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n",
 632                         rx_chn->udma_rchan_id, ret);
 633
 634         return ret;
 635 }
 636
 637 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
 638                                          u32 flow_num)
 639 {
 640         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
 641
 642         if (IS_ERR_OR_NULL(flow->udma_rflow))
 643                 return;
 644
 645         if (flow->ringrxfdq)
 646                 k3_ringacc_ring_free(flow->ringrxfdq);
 647
 648         if (flow->ringrx)
 649                 k3_ringacc_ring_free(flow->ringrx);
 650
 651         xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
 652         flow->udma_rflow = NULL;
 653         rx_chn->flows_ready--;
 654 }
 655
 656 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
 657                                     u32 flow_idx,
 658                                     struct k3_udma_glue_rx_flow_cfg *flow_cfg)
 659 {
 660         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
 661         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
 662         struct device *dev = rx_chn->common.dev;
 663         struct ti_sci_msg_rm_udmap_flow_cfg req;
 664         int rx_ring_id;
 665         int rx_ringfdq_id;
 666         int ret = 0;
 667
 668         flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax,
 669                                            flow->udma_rflow_id);
 670         if (IS_ERR(flow->udma_rflow)) {
 671                 ret = PTR_ERR(flow->udma_rflow);
 672                 dev_err(dev, "UDMAX rflow get err %d\n", ret);
 673                 return ret;
 674         }
 675
 676         if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) {
 677                 ret = -ENODEV;
 678                 goto err_rflow_put;
 679         }
 680
 681         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 682                 rx_ringfdq_id = flow->udma_rflow_id +
 683                                 xudma_get_rflow_ring_offset(rx_chn->common.udmax);
 684                 rx_ring_id = 0;
 685         } else {
 686                 rx_ring_id = flow_cfg->ring_rxq_id;
 687                 rx_ringfdq_id = flow_cfg->ring_rxfdq0_id;
 688         }
 689
 690         /* request and cfg rings */
 691         ret =  k3_ringacc_request_rings_pair(rx_chn->common.ringacc,
 692                                              rx_ringfdq_id, rx_ring_id,
 693                                              &flow->ringrxfdq,
 694                                              &flow->ringrx);
 695         if (ret) {
 696                 dev_err(dev, "Failed to get RX/RXFDQ rings %d\n", ret);
 697                 goto err_rflow_put;
 698         }
 699
 700         /* Set the dma_dev for the rings to be configured */
 701         flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn);
 702         flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev;
 703
 704         /* Set the ASEL value for DMA rings of PKTDMA */
 705         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 706                 flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel;
 707                 flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel;
 708         }
 709
 710         ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
 711         if (ret) {
 712                 dev_err(dev, "Failed to cfg ringrx %d\n", ret);
 713                 goto err_ringrxfdq_free;
 714         }
 715
 716         ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg);
 717         if (ret) {
 718                 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret);
 719                 goto err_ringrxfdq_free;
 720         }
 721
 722         if (rx_chn->remote) {
 723                 rx_ring_id = TI_SCI_RESOURCE_NULL;
 724                 rx_ringfdq_id = TI_SCI_RESOURCE_NULL;
 725         } else {
 726                 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
 727                 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
 728         }
 729
 730         memset(&req, 0, sizeof(req));
 731
 732         req.valid_params =
 733                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
 734                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
 735                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
 736                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
 737                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
 738                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
 739                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
 740                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
 741                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
 742                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
 743                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
 744                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
 745                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
 746         req.nav_id = tisci_rm->tisci_dev_id;
 747         req.flow_index = flow->udma_rflow_id;
 748         if (rx_chn->common.epib)
 749                 req.rx_einfo_present = 1;
 750         if (rx_chn->common.psdata_size)
 751                 req.rx_psinfo_present = 1;
 752         if (flow_cfg->rx_error_handling)
 753                 req.rx_error_handling = 1;
 754         req.rx_desc_type = 0;
 755         req.rx_dest_qnum = rx_ring_id;
 756         req.rx_src_tag_hi_sel = 0;
 757         req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel;
 758         req.rx_dest_tag_hi_sel = 0;
 759         req.rx_dest_tag_lo_sel = 0;
 760         req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
 761         req.rx_fdq1_qnum = rx_ringfdq_id;
 762         req.rx_fdq2_qnum = rx_ringfdq_id;
 763         req.rx_fdq3_qnum = rx_ringfdq_id;
 764
 765         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
 766         if (ret) {
 767                 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id,
 768                         ret);
 769                 goto err_ringrxfdq_free;
 770         }
 771
 772         rx_chn->flows_ready++;
 773         dev_dbg(dev, "flow%d config done. ready:%d\n",
 774                 flow->udma_rflow_id, rx_chn->flows_ready);
 775
 776         return 0;
 777
 778 err_ringrxfdq_free:
 779         k3_ringacc_ring_free(flow->ringrxfdq);
 780         k3_ringacc_ring_free(flow->ringrx);
 781
 782 err_rflow_put:
 783         xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
 784         flow->udma_rflow = NULL;
 785
 786         return ret;
 787 }
 788
 789 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn)
 790 {
 791         struct device *dev = chn->common.dev;
 792
 793         dev_dbg(dev, "dump_rx_chn:\n"
 794                 "udma_rchan_id: %d\n"
 795                 "src_thread: %08x\n"
 796                 "dst_thread: %08x\n"
 797                 "epib: %d\n"
 798                 "hdesc_size: %u\n"
 799                 "psdata_size: %u\n"
 800                 "swdata_size: %u\n"
 801                 "flow_id_base: %d\n"
 802                 "flow_num: %d\n",
 803                 chn->udma_rchan_id,
 804                 chn->common.src_thread,
 805                 chn->common.dst_thread,
 806                 chn->common.epib,
 807                 chn->common.hdesc_size,
 808                 chn->common.psdata_size,
 809                 chn->common.swdata_size,
 810                 chn->flow_id_base,
 811                 chn->flow_num);
 812 }
 813
 814 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn,
 815                                         char *mark)
 816 {
 817         struct device *dev = chn->common.dev;
 818
 819         dev_dbg(dev, "=== dump ===> %s\n", mark);
 820
 821         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
 822                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG));
 823         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
 824                 xudma_rchanrt_read(chn->udma_rchanx,
 825                                    UDMA_CHAN_RT_PEER_RT_EN_REG));
 826         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
 827                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_PCNT_REG));
 828         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
 829                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_BCNT_REG));
 830         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
 831                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_SBCNT_REG));
 832 }
 833
 834 static int
 835 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn,
 836                                struct k3_udma_glue_rx_channel_cfg *cfg)
 837 {
 838         int ret;
 839
 840         /* default rflow */
 841         if (cfg->flow_id_use_rxchan_id)
 842                 return 0;
 843
 844         /* not a GP rflows */
 845         if (rx_chn->flow_id_base != -1 &&
 846             !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
 847                 return 0;
 848
 849         /* Allocate range of GP rflows */
 850         ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax,
 851                                          rx_chn->flow_id_base,
 852                                          rx_chn->flow_num);
 853         if (ret < 0) {
 854                 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n",
 855                         rx_chn->flow_id_base, rx_chn->flow_num, ret);
 856                 return ret;
 857         }
 858         rx_chn->flow_id_base = ret;
 859
 860         return 0;
 861 }
 862
 863 static struct k3_udma_glue_rx_channel *
 864 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
 865                                  struct k3_udma_glue_rx_channel_cfg *cfg)
 866 {
 867         struct k3_udma_glue_rx_channel *rx_chn;
 868         struct psil_endpoint_config *ep_cfg;
 869         int ret, i;
 870
 871         if (cfg->flow_id_num <= 0)
 872                 return ERR_PTR(-EINVAL);
 873
 874         if (cfg->flow_id_num != 1 &&
 875             (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id))
 876                 return ERR_PTR(-EINVAL);
 877
 878         rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
 879         if (!rx_chn)
 880                 return ERR_PTR(-ENOMEM);
 881
 882         rx_chn->common.dev = dev;
 883         rx_chn->common.swdata_size = cfg->swdata_size;
 884         rx_chn->remote = false;
 885
 886         /* parse of udmap channel */
 887         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
 888                                         &rx_chn->common, false);
 889         if (ret)
 890                 goto err;
 891
 892         rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
 893                                                 rx_chn->common.psdata_size,
 894                                                 rx_chn->common.swdata_size);
 895
 896         ep_cfg = rx_chn->common.ep_config;
 897
 898         if (xudma_is_pktdma(rx_chn->common.udmax))
 899                 rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id;
 900         else
 901                 rx_chn->udma_rchan_id = -1;
 902
 903         /* request and cfg UDMAP RX channel */
 904         rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax,
 905                                               rx_chn->udma_rchan_id);
 906         if (IS_ERR(rx_chn->udma_rchanx)) {
 907                 ret = PTR_ERR(rx_chn->udma_rchanx);
 908                 dev_err(dev, "UDMAX rchanx get err %d\n", ret);
 909                 goto err;
 910         }
 911         rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
 912
 913         rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
 914         rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
 915         dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x",
 916                      rx_chn->udma_rchan_id, rx_chn->common.src_thread);
 917         ret = device_register(&rx_chn->common.chan_dev);
 918         if (ret) {
 919                 dev_err(dev, "Channel Device registration failed %d\n", ret);
 920                 rx_chn->common.chan_dev.parent = NULL;
 921                 goto err;
 922         }
 923
 924         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 925                 /* prepare the channel device as coherent */
 926                 rx_chn->common.chan_dev.dma_coherent = true;
 927                 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
 928                                              DMA_BIT_MASK(48));
 929         }
 930
 931         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 932                 int flow_start = cfg->flow_id_base;
 933                 int flow_end;
 934
 935                 if (flow_start == -1)
 936                         flow_start = ep_cfg->flow_start;
 937
 938                 flow_end = flow_start + cfg->flow_id_num - 1;
 939                 if (flow_start < ep_cfg->flow_start ||
 940                     flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) {
 941                         dev_err(dev, "Invalid flow range requested\n");
 942                         ret = -EINVAL;
 943                         goto err;
 944                 }
 945                 rx_chn->flow_id_base = flow_start;
 946         } else {
 947                 rx_chn->flow_id_base = cfg->flow_id_base;
 948
 949                 /* Use RX channel id as flow id: target dev can't generate flow_id */
 950                 if (cfg->flow_id_use_rxchan_id)
 951                         rx_chn->flow_id_base = rx_chn->udma_rchan_id;
 952         }
 953
 954         rx_chn->flow_num = cfg->flow_id_num;
 955
 956         rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
 957                                      sizeof(*rx_chn->flows), GFP_KERNEL);
 958         if (!rx_chn->flows) {
 959                 ret = -ENOMEM;
 960                 goto err;
 961         }
 962
 963         ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
 964         if (ret)
 965                 goto err;
 966
 967         for (i = 0; i < rx_chn->flow_num; i++)
 968                 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
 969
 970         /* request and cfg psi-l */
 971         rx_chn->common.dst_thread =
 972                         xudma_dev_get_psil_base(rx_chn->common.udmax) +
 973                         rx_chn->udma_rchan_id;
 974
 975         ret = k3_udma_glue_cfg_rx_chn(rx_chn);
 976         if (ret) {
 977                 dev_err(dev, "Failed to cfg rchan %d\n", ret);
 978                 goto err;
 979         }
 980
 981         /* init default RX flow only if flow_num = 1 */
 982         if (cfg->def_flow_cfg) {
 983                 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg);
 984                 if (ret)
 985                         goto err;
 986         }
 987
 988         k3_udma_glue_dump_rx_chn(rx_chn);
 989
 990         return rx_chn;
 991
 992 err:
 993         k3_udma_glue_release_rx_chn(rx_chn);
 994         return ERR_PTR(ret);
 995 }
 996
 997 static struct k3_udma_glue_rx_channel *
 998 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name,
 999                                    struct k3_udma_glue_rx_channel_cfg *cfg)
1000 {
1001         struct k3_udma_glue_rx_channel *rx_chn;
1002         int ret, i;
1003
1004         if (cfg->flow_id_num <= 0 ||
1005             cfg->flow_id_use_rxchan_id ||
1006             cfg->def_flow_cfg ||
1007             cfg->flow_id_base < 0)
1008                 return ERR_PTR(-EINVAL);
1009
1010         /*
1011          * Remote RX channel is under control of Remote CPU core, so
1012          * Linux can only request and manipulate by dedicated RX flows
1013          */
1014
1015         rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
1016         if (!rx_chn)
1017                 return ERR_PTR(-ENOMEM);
1018
1019         rx_chn->common.dev = dev;
1020         rx_chn->common.swdata_size = cfg->swdata_size;
1021         rx_chn->remote = true;
1022         rx_chn->udma_rchan_id = -1;
1023         rx_chn->flow_num = cfg->flow_id_num;
1024         rx_chn->flow_id_base = cfg->flow_id_base;
1025         rx_chn->psil_paired = false;
1026
1027         /* parse of udmap channel */
1028         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
1029                                         &rx_chn->common, false);
1030         if (ret)
1031                 goto err;
1032
1033         rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
1034                                                 rx_chn->common.psdata_size,
1035                                                 rx_chn->common.swdata_size);
1036
1037         rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
1038                                      sizeof(*rx_chn->flows), GFP_KERNEL);
1039         if (!rx_chn->flows) {
1040                 ret = -ENOMEM;
1041                 goto err;
1042         }
1043
1044         rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
1045         rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
1046         dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x",
1047                      rx_chn->common.src_thread);
1048         ret = device_register(&rx_chn->common.chan_dev);
1049         if (ret) {
1050                 dev_err(dev, "Channel Device registration failed %d\n", ret);
1051                 rx_chn->common.chan_dev.parent = NULL;
1052                 goto err;
1053         }
1054
1055         if (xudma_is_pktdma(rx_chn->common.udmax)) {
1056                 /* prepare the channel device as coherent */
1057                 rx_chn->common.chan_dev.dma_coherent = true;
1058                 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
1059                                              DMA_BIT_MASK(48));
1060         }
1061
1062         ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
1063         if (ret)
1064                 goto err;
1065
1066         for (i = 0; i < rx_chn->flow_num; i++)
1067                 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
1068
1069         k3_udma_glue_dump_rx_chn(rx_chn);
1070
1071         return rx_chn;
1072
1073 err:
1074         k3_udma_glue_release_rx_chn(rx_chn);
1075         return ERR_PTR(ret);
1076 }
1077
1078 struct k3_udma_glue_rx_channel *
1079 k3_udma_glue_request_rx_chn(struct device *dev, const char *name,
1080                             struct k3_udma_glue_rx_channel_cfg *cfg)
1081 {
1082         if (cfg->remote)
1083                 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg);
1084         else
1085                 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg);
1086 }
1087 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn);
1088
1089 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1090 {
1091         int i;
1092
1093         if (IS_ERR_OR_NULL(rx_chn->common.udmax))
1094                 return;
1095
1096         if (rx_chn->psil_paired) {
1097                 xudma_navss_psil_unpair(rx_chn->common.udmax,
1098                                         rx_chn->common.src_thread,
1099                                         rx_chn->common.dst_thread);
1100                 rx_chn->psil_paired = false;
1101         }
1102
1103         for (i = 0; i < rx_chn->flow_num; i++)
1104                 k3_udma_glue_release_rx_flow(rx_chn, i);
1105
1106         if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
1107                 xudma_free_gp_rflow_range(rx_chn->common.udmax,
1108                                           rx_chn->flow_id_base,
1109                                           rx_chn->flow_num);
1110
1111         if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
1112                 xudma_rchan_put(rx_chn->common.udmax,
1113                                 rx_chn->udma_rchanx);
1114
1115         if (rx_chn->common.chan_dev.parent) {
1116                 device_unregister(&rx_chn->common.chan_dev);
1117                 rx_chn->common.chan_dev.parent = NULL;
1118         }
1119 }
1120 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn);
1121
1122 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn,
1123                               u32 flow_idx,
1124                               struct k3_udma_glue_rx_flow_cfg *flow_cfg)
1125 {
1126         if (flow_idx >= rx_chn->flow_num)
1127                 return -EINVAL;
1128
1129         return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg);
1130 }
1131 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init);
1132
1133 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn,
1134                                     u32 flow_idx)
1135 {
1136         struct k3_udma_glue_rx_flow *flow;
1137
1138         if (flow_idx >= rx_chn->flow_num)
1139                 return -EINVAL;
1140
1141         flow = &rx_chn->flows[flow_idx];
1142
1143         return k3_ringacc_get_ring_id(flow->ringrxfdq);
1144 }
1145 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id);
1146
1147 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn)
1148 {
1149         return rx_chn->flow_id_base;
1150 }
1151 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base);
1152
1153 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
1154                                 u32 flow_idx)
1155 {
1156         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1157         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1158         struct device *dev = rx_chn->common.dev;
1159         struct ti_sci_msg_rm_udmap_flow_cfg req;
1160         int rx_ring_id;
1161         int rx_ringfdq_id;
1162         int ret = 0;
1163
1164         if (!rx_chn->remote)
1165                 return -EINVAL;
1166
1167         rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
1168         rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
1169
1170         memset(&req, 0, sizeof(req));
1171
1172         req.valid_params =
1173                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1174                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1175                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1176                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1177                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1178         req.nav_id = tisci_rm->tisci_dev_id;
1179         req.flow_index = flow->udma_rflow_id;
1180         req.rx_dest_qnum = rx_ring_id;
1181         req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
1182         req.rx_fdq1_qnum = rx_ringfdq_id;
1183         req.rx_fdq2_qnum = rx_ringfdq_id;
1184         req.rx_fdq3_qnum = rx_ringfdq_id;
1185
1186         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1187         if (ret) {
1188                 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id,
1189                         ret);
1190         }
1191
1192         return ret;
1193 }
1194 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable);
1195
1196 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
1197                                  u32 flow_idx)
1198 {
1199         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1200         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1201         struct device *dev = rx_chn->common.dev;
1202         struct ti_sci_msg_rm_udmap_flow_cfg req;
1203         int ret = 0;
1204
1205         if (!rx_chn->remote)
1206                 return -EINVAL;
1207
1208         memset(&req, 0, sizeof(req));
1209         req.valid_params =
1210                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1211                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1212                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1213                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1214                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1215         req.nav_id = tisci_rm->tisci_dev_id;
1216         req.flow_index = flow->udma_rflow_id;
1217         req.rx_dest_qnum = TI_SCI_RESOURCE_NULL;
1218         req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL;
1219         req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL;
1220         req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL;
1221         req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL;
1222
1223         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1224         if (ret) {
1225                 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id,
1226                         ret);
1227         }
1228
1229         return ret;
1230 }
1231 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable);
1232
1233 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1234 {
1235         int ret;
1236
1237         if (rx_chn->remote)
1238                 return -EINVAL;
1239
1240         if (rx_chn->flows_ready < rx_chn->flow_num)
1241                 return -EINVAL;
1242
1243         ret = xudma_navss_psil_pair(rx_chn->common.udmax,
1244                                     rx_chn->common.src_thread,
1245                                     rx_chn->common.dst_thread);
1246         if (ret) {
1247                 dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret);
1248                 return ret;
1249         }
1250
1251         rx_chn->psil_paired = true;
1252
1253         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG,
1254                             UDMA_CHAN_RT_CTL_EN);
1255
1256         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1257                             UDMA_PEER_RT_EN_ENABLE);
1258
1259         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en");
1260         return 0;
1261 }
1262 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn);
1263
1264 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1265 {
1266         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1");
1267
1268         xudma_rchanrt_write(rx_chn->udma_rchanx,
1269                             UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
1270         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0);
1271
1272         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2");
1273
1274         if (rx_chn->psil_paired) {
1275                 xudma_navss_psil_unpair(rx_chn->common.udmax,
1276                                         rx_chn->common.src_thread,
1277                                         rx_chn->common.dst_thread);
1278                 rx_chn->psil_paired = false;
1279         }
1280 }
1281 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn);
1282
1283 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1284                                bool sync)
1285 {
1286         int i = 0;
1287         u32 val;
1288
1289         if (rx_chn->remote)
1290                 return;
1291
1292         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1");
1293
1294         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1295                             UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN);
1296
1297         val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG);
1298
1299         while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
1300                 val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1301                                          UDMA_CHAN_RT_CTL_REG);
1302                 udelay(1);
1303                 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
1304                         dev_err(rx_chn->common.dev, "RX tdown timeout\n");
1305                         break;
1306                 }
1307                 i++;
1308         }
1309
1310         val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1311                                  UDMA_CHAN_RT_PEER_RT_EN_REG);
1312         if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
1313                 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n");
1314         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2");
1315 }
1316 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn);
1317
1318 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1319                 u32 flow_num, void *data,
1320                 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq)
1321 {
1322         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1323         struct device *dev = rx_chn->common.dev;
1324         dma_addr_t desc_dma;
1325         int occ_rx, i, ret;
1326
1327         /* reset RXCQ as it is not input for udma - expected to be empty */
1328         occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
1329         dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
1330
1331         /* Skip RX FDQ in case one FDQ is used for the set of flows */
1332         if (skip_fdq)
1333                 goto do_reset;
1334
1335         /*
1336          * RX FDQ reset need to be special way as it is input for udma and its
1337          * state cached by udma, so:
1338          * 1) save RX FDQ occ
1339          * 2) clean up RX FDQ and call callback .cleanup() for each desc
1340          * 3) reset RX FDQ in a special way
1341          */
1342         occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq);
1343         dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx);
1344
1345         for (i = 0; i < occ_rx; i++) {
1346                 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
1347                 if (ret) {
1348                         if (ret != -ENODATA)
1349                                 dev_err(dev, "RX reset pop %d\n", ret);
1350                         break;
1351                 }
1352                 cleanup(data, desc_dma);
1353         }
1354
1355         k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
1356
1357 do_reset:
1358         k3_ringacc_ring_reset(flow->ringrx);
1359 }
1360 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn);
1361
1362 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1363                              u32 flow_num, struct cppi5_host_desc_t *desc_rx,
1364                              dma_addr_t desc_dma)
1365 {
1366         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1367
1368         return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma);
1369 }
1370 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn);
1371
1372 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1373                             u32 flow_num, dma_addr_t *desc_dma)
1374 {
1375         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1376
1377         return k3_ringacc_ring_pop(flow->ringrx, desc_dma);
1378 }
1379 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn);
1380
1381 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
1382                             u32 flow_num)
1383 {
1384         struct k3_udma_glue_rx_flow *flow;
1385
1386         flow = &rx_chn->flows[flow_num];
1387
1388         if (xudma_is_pktdma(rx_chn->common.udmax)) {
1389                 flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax,
1390                                                         flow->udma_rflow_id);
1391         } else {
1392                 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
1393         }
1394
1395         return flow->virq;
1396 }
1397 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq);
1398
1399 struct device *
1400         k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn)
1401 {
1402         if (xudma_is_pktdma(rx_chn->common.udmax) &&
1403             (rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15))
1404                 return &rx_chn->common.chan_dev;
1405
1406         return xudma_get_device(rx_chn->common.udmax);
1407 }
1408 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device);
1409
1410 void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn,
1411                                        dma_addr_t *addr)
1412 {
1413         if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1414             !rx_chn->common.atype_asel)
1415                 return;
1416
1417         *addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
1418 }
1419 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr);
1420
1421 void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn,
1422                                        dma_addr_t *addr)
1423 {
1424         if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1425             !rx_chn->common.atype_asel)
1426                 return;
1427
1428         *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1429 }
1430 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr);
1431
1432 static int __init k3_udma_glue_class_init(void)
1433 {
1434         return class_register(&k3_udma_glue_devclass);
1435 }
1436 arch_initcall(k3_udma_glue_class_init);