drivers/dma/ti/k3-udma-glue.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * K3 NAVSS DMA glue interface
   4  *
   5  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/atomic.h>
  11 #include <linux/delay.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/io.h>
  14 #include <linux/init.h>
  15 #include <linux/of.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/soc/ti/k3-ringacc.h>
  18 #include <linux/dma/ti-cppi5.h>
  19 #include <linux/dma/k3-udma-glue.h>
  20
  21 #include "k3-udma.h"
  22 #include "k3-psil-priv.h"
  23
  24 struct k3_udma_glue_common {
  25         struct device *dev;
  26         struct device chan_dev;
  27         struct udma_dev *udmax;
  28         const struct udma_tisci_rm *tisci_rm;
  29         struct k3_ringacc *ringacc;
  30         u32 src_thread;
  31         u32 dst_thread;
  32
  33         u32  hdesc_size;
  34         bool epib;
  35         u32  psdata_size;
  36         u32  swdata_size;
  37         u32  atype_asel;
  38         struct psil_endpoint_config *ep_config;
  39 };
  40
  41 struct k3_udma_glue_tx_channel {
  42         struct k3_udma_glue_common common;
  43
  44         struct udma_tchan *udma_tchanx;
  45         int udma_tchan_id;
  46
  47         struct k3_ring *ringtx;
  48         struct k3_ring *ringtxcq;
  49
  50         bool psil_paired;
  51
  52         int virq;
  53
  54         atomic_t free_pkts;
  55         bool tx_pause_on_err;
  56         bool tx_filt_einfo;
  57         bool tx_filt_pswords;
  58         bool tx_supr_tdpkt;
  59
  60         int udma_tflow_id;
  61 };
  62
  63 struct k3_udma_glue_rx_flow {
  64         struct udma_rflow *udma_rflow;
  65         int udma_rflow_id;
  66         struct k3_ring *ringrx;
  67         struct k3_ring *ringrxfdq;
  68
  69         int virq;
  70 };
  71
  72 struct k3_udma_glue_rx_channel {
  73         struct k3_udma_glue_common common;
  74
  75         struct udma_rchan *udma_rchanx;
  76         int udma_rchan_id;
  77         bool remote;
  78
  79         bool psil_paired;
  80
  81         u32  swdata_size;
  82         int  flow_id_base;
  83
  84         struct k3_udma_glue_rx_flow *flows;
  85         u32 flow_num;
  86         u32 flows_ready;
  87 };
  88
  89 static void k3_udma_chan_dev_release(struct device *dev)
  90 {
  91         /* The struct containing the device is devm managed */
  92 }
  93
  94 static struct class k3_udma_glue_devclass = {
  95         .name           = "k3_udma_glue_chan",
  96         .dev_release    = k3_udma_chan_dev_release,
  97 };
  98
  99 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000
 100
 101 static int of_k3_udma_glue_parse(struct device_node *udmax_np,
 102                                  struct k3_udma_glue_common *common)
 103 {
 104         common->udmax = of_xudma_dev_get(udmax_np, NULL);
 105         if (IS_ERR(common->udmax))
 106                 return PTR_ERR(common->udmax);
 107
 108         common->ringacc = xudma_get_ringacc(common->udmax);
 109         common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
 110
 111         return 0;
 112 }
 113
 114 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
 115                 const char *name, struct k3_udma_glue_common *common,
 116                 bool tx_chn)
 117 {
 118         struct of_phandle_args dma_spec;
 119         u32 thread_id;
 120         int ret = 0;
 121         int index;
 122
 123         if (unlikely(!name))
 124                 return -EINVAL;
 125
 126         index = of_property_match_string(chn_np, "dma-names", name);
 127         if (index < 0)
 128                 return index;
 129
 130         if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index,
 131                                        &dma_spec))
 132                 return -ENOENT;
 133
 134         ret = of_k3_udma_glue_parse(dma_spec.np, common);
 135         if (ret)
 136                 goto out_put_spec;
 137
 138         thread_id = dma_spec.args[0];
 139         if (dma_spec.args_count == 2) {
 140                 if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) {
 141                         dev_err(common->dev, "Invalid channel atype: %u\n",
 142                                 dma_spec.args[1]);
 143                         ret = -EINVAL;
 144                         goto out_put_spec;
 145                 }
 146                 if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) {
 147                         dev_err(common->dev, "Invalid channel asel: %u\n",
 148                                 dma_spec.args[1]);
 149                         ret = -EINVAL;
 150                         goto out_put_spec;
 151                 }
 152
 153                 common->atype_asel = dma_spec.args[1];
 154         }
 155
 156         if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
 157                 ret = -EINVAL;
 158                 goto out_put_spec;
 159         }
 160
 161         if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
 162                 ret = -EINVAL;
 163                 goto out_put_spec;
 164         }
 165
 166         /* get psil endpoint config */
 167         common->ep_config = psil_get_ep_config(thread_id);
 168         if (IS_ERR(common->ep_config)) {
 169                 dev_err(common->dev,
 170                         "No configuration for psi-l thread 0x%04x\n",
 171                         thread_id);
 172                 ret = PTR_ERR(common->ep_config);
 173                 goto out_put_spec;
 174         }
 175
 176         common->epib = common->ep_config->needs_epib;
 177         common->psdata_size = common->ep_config->psd_size;
 178
 179         if (tx_chn)
 180                 common->dst_thread = thread_id;
 181         else
 182                 common->src_thread = thread_id;
 183
 184 out_put_spec:
 185         of_node_put(dma_spec.np);
 186         return ret;
 187 };
 188
 189 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 190 {
 191         struct device *dev = tx_chn->common.dev;
 192
 193         dev_dbg(dev, "dump_tx_chn:\n"
 194                 "udma_tchan_id: %d\n"
 195                 "src_thread: %08x\n"
 196                 "dst_thread: %08x\n",
 197                 tx_chn->udma_tchan_id,
 198                 tx_chn->common.src_thread,
 199                 tx_chn->common.dst_thread);
 200 }
 201
 202 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn,
 203                                         char *mark)
 204 {
 205         struct device *dev = chn->common.dev;
 206
 207         dev_dbg(dev, "=== dump ===> %s\n", mark);
 208         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
 209                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG));
 210         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
 211                 xudma_tchanrt_read(chn->udma_tchanx,
 212                                    UDMA_CHAN_RT_PEER_RT_EN_REG));
 213         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
 214                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_PCNT_REG));
 215         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
 216                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_BCNT_REG));
 217         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
 218                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_SBCNT_REG));
 219 }
 220
 221 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 222 {
 223         const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm;
 224         struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
 225
 226         memset(&req, 0, sizeof(req));
 227
 228         req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
 229                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
 230                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
 231                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
 232                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
 233                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
 234                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
 235                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
 236         req.nav_id = tisci_rm->tisci_dev_id;
 237         req.index = tx_chn->udma_tchan_id;
 238         if (tx_chn->tx_pause_on_err)
 239                 req.tx_pause_on_err = 1;
 240         if (tx_chn->tx_filt_einfo)
 241                 req.tx_filt_einfo = 1;
 242         if (tx_chn->tx_filt_pswords)
 243                 req.tx_filt_pswords = 1;
 244         req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
 245         if (tx_chn->tx_supr_tdpkt)
 246                 req.tx_supr_tdpkt = 1;
 247         req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
 248         req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 249         req.tx_atype = tx_chn->common.atype_asel;
 250
 251         return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
 252 }
 253
 254 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
 255                 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg)
 256 {
 257         struct k3_udma_glue_tx_channel *tx_chn;
 258         int ret;
 259
 260         tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL);
 261         if (!tx_chn)
 262                 return ERR_PTR(-ENOMEM);
 263
 264         tx_chn->common.dev = dev;
 265         tx_chn->common.swdata_size = cfg->swdata_size;
 266         tx_chn->tx_pause_on_err = cfg->tx_pause_on_err;
 267         tx_chn->tx_filt_einfo = cfg->tx_filt_einfo;
 268         tx_chn->tx_filt_pswords = cfg->tx_filt_pswords;
 269         tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt;
 270
 271         /* parse of udmap channel */
 272         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
 273                                         &tx_chn->common, true);
 274         if (ret)
 275                 goto err;
 276
 277         tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib,
 278                                                 tx_chn->common.psdata_size,
 279                                                 tx_chn->common.swdata_size);
 280
 281         if (xudma_is_pktdma(tx_chn->common.udmax))
 282                 tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id;
 283         else
 284                 tx_chn->udma_tchan_id = -1;
 285
 286         /* request and cfg UDMAP TX channel */
 287         tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax,
 288                                               tx_chn->udma_tchan_id);
 289         if (IS_ERR(tx_chn->udma_tchanx)) {
 290                 ret = PTR_ERR(tx_chn->udma_tchanx);
 291                 dev_err(dev, "UDMAX tchanx get err %d\n", ret);
 292                 goto err;
 293         }
 294         tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
 295
 296         tx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
 297         tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax);
 298         dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x",
 299                      tx_chn->udma_tchan_id, tx_chn->common.dst_thread);
 300         ret = device_register(&tx_chn->common.chan_dev);
 301         if (ret) {
 302                 dev_err(dev, "Channel Device registration failed %d\n", ret);
 303                 put_device(&tx_chn->common.chan_dev);
 304                 tx_chn->common.chan_dev.parent = NULL;
 305                 goto err;
 306         }
 307
 308         if (xudma_is_pktdma(tx_chn->common.udmax)) {
 309                 /* prepare the channel device as coherent */
 310                 tx_chn->common.chan_dev.dma_coherent = true;
 311                 dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
 312                                              DMA_BIT_MASK(48));
 313         }
 314
 315         atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
 316
 317         if (xudma_is_pktdma(tx_chn->common.udmax))
 318                 tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id;
 319         else
 320                 tx_chn->udma_tflow_id = tx_chn->udma_tchan_id;
 321
 322         /* request and cfg rings */
 323         ret =  k3_ringacc_request_rings_pair(tx_chn->common.ringacc,
 324                                              tx_chn->udma_tflow_id, -1,
 325                                              &tx_chn->ringtx,
 326                                              &tx_chn->ringtxcq);
 327         if (ret) {
 328                 dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret);
 329                 goto err;
 330         }
 331
 332         /* Set the dma_dev for the rings to be configured */
 333         cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn);
 334         cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev;
 335
 336         /* Set the ASEL value for DMA rings of PKTDMA */
 337         if (xudma_is_pktdma(tx_chn->common.udmax)) {
 338                 cfg->tx_cfg.asel = tx_chn->common.atype_asel;
 339                 cfg->txcq_cfg.asel = tx_chn->common.atype_asel;
 340         }
 341
 342         ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
 343         if (ret) {
 344                 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
 345                 goto err;
 346         }
 347
 348         ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg);
 349         if (ret) {
 350                 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
 351                 goto err;
 352         }
 353
 354         /* request and cfg psi-l */
 355         tx_chn->common.src_thread =
 356                         xudma_dev_get_psil_base(tx_chn->common.udmax) +
 357                         tx_chn->udma_tchan_id;
 358
 359         ret = k3_udma_glue_cfg_tx_chn(tx_chn);
 360         if (ret) {
 361                 dev_err(dev, "Failed to cfg tchan %d\n", ret);
 362                 goto err;
 363         }
 364
 365         k3_udma_glue_dump_tx_chn(tx_chn);
 366
 367         return tx_chn;
 368
 369 err:
 370         k3_udma_glue_release_tx_chn(tx_chn);
 371         return ERR_PTR(ret);
 372 }
 373 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn);
 374
 375 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 376 {
 377         if (tx_chn->psil_paired) {
 378                 xudma_navss_psil_unpair(tx_chn->common.udmax,
 379                                         tx_chn->common.src_thread,
 380                                         tx_chn->common.dst_thread);
 381                 tx_chn->psil_paired = false;
 382         }
 383
 384         if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx))
 385                 xudma_tchan_put(tx_chn->common.udmax,
 386                                 tx_chn->udma_tchanx);
 387
 388         if (tx_chn->ringtxcq)
 389                 k3_ringacc_ring_free(tx_chn->ringtxcq);
 390
 391         if (tx_chn->ringtx)
 392                 k3_ringacc_ring_free(tx_chn->ringtx);
 393
 394         if (tx_chn->common.chan_dev.parent) {
 395                 device_unregister(&tx_chn->common.chan_dev);
 396                 tx_chn->common.chan_dev.parent = NULL;
 397         }
 398 }
 399 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn);
 400
 401 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 402                              struct cppi5_host_desc_t *desc_tx,
 403                              dma_addr_t desc_dma)
 404 {
 405         u32 ringtxcq_id;
 406
 407         if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0))
 408                 return -ENOMEM;
 409
 410         ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 411         cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id);
 412
 413         return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma);
 414 }
 415 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn);
 416
 417 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 418                             dma_addr_t *desc_dma)
 419 {
 420         int ret;
 421
 422         ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma);
 423         if (!ret)
 424                 atomic_inc(&tx_chn->free_pkts);
 425
 426         return ret;
 427 }
 428 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn);
 429
 430 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 431 {
 432         int ret;
 433
 434         ret = xudma_navss_psil_pair(tx_chn->common.udmax,
 435                                     tx_chn->common.src_thread,
 436                                     tx_chn->common.dst_thread);
 437         if (ret) {
 438                 dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret);
 439                 return ret;
 440         }
 441
 442         tx_chn->psil_paired = true;
 443
 444         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
 445                             UDMA_PEER_RT_EN_ENABLE);
 446
 447         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
 448                             UDMA_CHAN_RT_CTL_EN);
 449
 450         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en");
 451         return 0;
 452 }
 453 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn);
 454
 455 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 456 {
 457         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1");
 458
 459         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 0);
 460
 461         xudma_tchanrt_write(tx_chn->udma_tchanx,
 462                             UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
 463         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2");
 464
 465         if (tx_chn->psil_paired) {
 466                 xudma_navss_psil_unpair(tx_chn->common.udmax,
 467                                         tx_chn->common.src_thread,
 468                                         tx_chn->common.dst_thread);
 469                 tx_chn->psil_paired = false;
 470         }
 471 }
 472 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn);
 473
 474 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 475                                bool sync)
 476 {
 477         int i = 0;
 478         u32 val;
 479
 480         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1");
 481
 482         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
 483                             UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN);
 484
 485         val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG);
 486
 487         while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
 488                 val = xudma_tchanrt_read(tx_chn->udma_tchanx,
 489                                          UDMA_CHAN_RT_CTL_REG);
 490                 udelay(1);
 491                 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
 492                         dev_err(tx_chn->common.dev, "TX tdown timeout\n");
 493                         break;
 494                 }
 495                 i++;
 496         }
 497
 498         val = xudma_tchanrt_read(tx_chn->udma_tchanx,
 499                                  UDMA_CHAN_RT_PEER_RT_EN_REG);
 500         if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
 501                 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n");
 502         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2");
 503 }
 504 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn);
 505
 506 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 507                                void *data,
 508                                void (*cleanup)(void *data, dma_addr_t desc_dma))
 509 {
 510         struct device *dev = tx_chn->common.dev;
 511         dma_addr_t desc_dma;
 512         int occ_tx, i, ret;
 513
 514         /*
 515          * TXQ reset need to be special way as it is input for udma and its
 516          * state cached by udma, so:
 517          * 1) save TXQ occ
 518          * 2) clean up TXQ and call callback .cleanup() for each desc
 519          * 3) reset TXQ in a special way
 520          */
 521         occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
 522         dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx);
 523
 524         for (i = 0; i < occ_tx; i++) {
 525                 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
 526                 if (ret) {
 527                         if (ret != -ENODATA)
 528                                 dev_err(dev, "TX reset pop %d\n", ret);
 529                         break;
 530                 }
 531                 cleanup(data, desc_dma);
 532         }
 533
 534         /* reset TXCQ as it is not input for udma - expected to be empty */
 535         k3_ringacc_ring_reset(tx_chn->ringtxcq);
 536         k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
 537 }
 538 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn);
 539
 540 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn)
 541 {
 542         return tx_chn->common.hdesc_size;
 543 }
 544 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size);
 545
 546 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn)
 547 {
 548         return k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 549 }
 550 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id);
 551
 552 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn)
 553 {
 554         if (xudma_is_pktdma(tx_chn->common.udmax)) {
 555                 tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax,
 556                                                           tx_chn->udma_tflow_id);
 557         } else {
 558                 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
 559         }
 560
 561         if (!tx_chn->virq)
 562                 return -ENXIO;
 563
 564         return tx_chn->virq;
 565 }
 566 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq);
 567
 568 struct device *
 569         k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn)
 570 {
 571         if (xudma_is_pktdma(tx_chn->common.udmax) &&
 572             (tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15))
 573                 return &tx_chn->common.chan_dev;
 574
 575         return xudma_get_device(tx_chn->common.udmax);
 576 }
 577 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device);
 578
 579 void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn,
 580                                        dma_addr_t *addr)
 581 {
 582         if (!xudma_is_pktdma(tx_chn->common.udmax) ||
 583             !tx_chn->common.atype_asel)
 584                 return;
 585
 586         *addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
 587 }
 588 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr);
 589
 590 void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn,
 591                                        dma_addr_t *addr)
 592 {
 593         if (!xudma_is_pktdma(tx_chn->common.udmax) ||
 594             !tx_chn->common.atype_asel)
 595                 return;
 596
 597         *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
 598 }
 599 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr);
 600
 601 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
 602 {
 603         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
 604         struct ti_sci_msg_rm_udmap_rx_ch_cfg req;
 605         int ret;
 606
 607         memset(&req, 0, sizeof(req));
 608
 609         req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
 610                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
 611                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
 612                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
 613
 614         req.nav_id = tisci_rm->tisci_dev_id;
 615         req.index = rx_chn->udma_rchan_id;
 616         req.rx_fetch_size = rx_chn->common.hdesc_size >> 2;
 617         /*
 618          * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw
 619          * and udmax impl, so just configure it to invalid value.
 620          * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
 621          */
 622         req.rxcq_qnum = 0xFFFF;
 623         if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num &&
 624             rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
 625                 /* Default flow + extra ones */
 626                 req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
 627                                     TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
 628                 req.flowid_start = rx_chn->flow_id_base;
 629                 req.flowid_cnt = rx_chn->flow_num;
 630         }
 631         req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
 632         req.rx_atype = rx_chn->common.atype_asel;
 633
 634         ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
 635         if (ret)
 636                 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n",
 637                         rx_chn->udma_rchan_id, ret);
 638
 639         return ret;
 640 }
 641
 642 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
 643                                          u32 flow_num)
 644 {
 645         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
 646
 647         if (IS_ERR_OR_NULL(flow->udma_rflow))
 648                 return;
 649
 650         if (flow->ringrxfdq)
 651                 k3_ringacc_ring_free(flow->ringrxfdq);
 652
 653         if (flow->ringrx)
 654                 k3_ringacc_ring_free(flow->ringrx);
 655
 656         xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
 657         flow->udma_rflow = NULL;
 658         rx_chn->flows_ready--;
 659 }
 660
 661 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
 662                                     u32 flow_idx,
 663                                     struct k3_udma_glue_rx_flow_cfg *flow_cfg)
 664 {
 665         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
 666         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
 667         struct device *dev = rx_chn->common.dev;
 668         struct ti_sci_msg_rm_udmap_flow_cfg req;
 669         int rx_ring_id;
 670         int rx_ringfdq_id;
 671         int ret = 0;
 672
 673         flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax,
 674                                            flow->udma_rflow_id);
 675         if (IS_ERR(flow->udma_rflow)) {
 676                 ret = PTR_ERR(flow->udma_rflow);
 677                 dev_err(dev, "UDMAX rflow get err %d\n", ret);
 678                 return ret;
 679         }
 680
 681         if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) {
 682                 ret = -ENODEV;
 683                 goto err_rflow_put;
 684         }
 685
 686         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 687                 rx_ringfdq_id = flow->udma_rflow_id +
 688                                 xudma_get_rflow_ring_offset(rx_chn->common.udmax);
 689                 rx_ring_id = 0;
 690         } else {
 691                 rx_ring_id = flow_cfg->ring_rxq_id;
 692                 rx_ringfdq_id = flow_cfg->ring_rxfdq0_id;
 693         }
 694
 695         /* request and cfg rings */
 696         ret =  k3_ringacc_request_rings_pair(rx_chn->common.ringacc,
 697                                              rx_ringfdq_id, rx_ring_id,
 698                                              &flow->ringrxfdq,
 699                                              &flow->ringrx);
 700         if (ret) {
 701                 dev_err(dev, "Failed to get RX/RXFDQ rings %d\n", ret);
 702                 goto err_rflow_put;
 703         }
 704
 705         /* Set the dma_dev for the rings to be configured */
 706         flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn);
 707         flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev;
 708
 709         /* Set the ASEL value for DMA rings of PKTDMA */
 710         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 711                 flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel;
 712                 flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel;
 713         }
 714
 715         ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
 716         if (ret) {
 717                 dev_err(dev, "Failed to cfg ringrx %d\n", ret);
 718                 goto err_ringrxfdq_free;
 719         }
 720
 721         ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg);
 722         if (ret) {
 723                 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret);
 724                 goto err_ringrxfdq_free;
 725         }
 726
 727         if (rx_chn->remote) {
 728                 rx_ring_id = TI_SCI_RESOURCE_NULL;
 729                 rx_ringfdq_id = TI_SCI_RESOURCE_NULL;
 730         } else {
 731                 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
 732                 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
 733         }
 734
 735         memset(&req, 0, sizeof(req));
 736
 737         req.valid_params =
 738                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
 739                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
 740                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
 741                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
 742                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
 743                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
 744                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
 745                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
 746                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
 747                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
 748                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
 749                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
 750                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
 751         req.nav_id = tisci_rm->tisci_dev_id;
 752         req.flow_index = flow->udma_rflow_id;
 753         if (rx_chn->common.epib)
 754                 req.rx_einfo_present = 1;
 755         if (rx_chn->common.psdata_size)
 756                 req.rx_psinfo_present = 1;
 757         if (flow_cfg->rx_error_handling)
 758                 req.rx_error_handling = 1;
 759         req.rx_desc_type = 0;
 760         req.rx_dest_qnum = rx_ring_id;
 761         req.rx_src_tag_hi_sel = 0;
 762         req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel;
 763         req.rx_dest_tag_hi_sel = 0;
 764         req.rx_dest_tag_lo_sel = 0;
 765         req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
 766         req.rx_fdq1_qnum = rx_ringfdq_id;
 767         req.rx_fdq2_qnum = rx_ringfdq_id;
 768         req.rx_fdq3_qnum = rx_ringfdq_id;
 769
 770         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
 771         if (ret) {
 772                 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id,
 773                         ret);
 774                 goto err_ringrxfdq_free;
 775         }
 776
 777         rx_chn->flows_ready++;
 778         dev_dbg(dev, "flow%d config done. ready:%d\n",
 779                 flow->udma_rflow_id, rx_chn->flows_ready);
 780
 781         return 0;
 782
 783 err_ringrxfdq_free:
 784         k3_ringacc_ring_free(flow->ringrxfdq);
 785         k3_ringacc_ring_free(flow->ringrx);
 786
 787 err_rflow_put:
 788         xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
 789         flow->udma_rflow = NULL;
 790
 791         return ret;
 792 }
 793
 794 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn)
 795 {
 796         struct device *dev = chn->common.dev;
 797
 798         dev_dbg(dev, "dump_rx_chn:\n"
 799                 "udma_rchan_id: %d\n"
 800                 "src_thread: %08x\n"
 801                 "dst_thread: %08x\n"
 802                 "epib: %d\n"
 803                 "hdesc_size: %u\n"
 804                 "psdata_size: %u\n"
 805                 "swdata_size: %u\n"
 806                 "flow_id_base: %d\n"
 807                 "flow_num: %d\n",
 808                 chn->udma_rchan_id,
 809                 chn->common.src_thread,
 810                 chn->common.dst_thread,
 811                 chn->common.epib,
 812                 chn->common.hdesc_size,
 813                 chn->common.psdata_size,
 814                 chn->common.swdata_size,
 815                 chn->flow_id_base,
 816                 chn->flow_num);
 817 }
 818
 819 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn,
 820                                         char *mark)
 821 {
 822         struct device *dev = chn->common.dev;
 823
 824         dev_dbg(dev, "=== dump ===> %s\n", mark);
 825
 826         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
 827                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG));
 828         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
 829                 xudma_rchanrt_read(chn->udma_rchanx,
 830                                    UDMA_CHAN_RT_PEER_RT_EN_REG));
 831         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
 832                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_PCNT_REG));
 833         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
 834                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_BCNT_REG));
 835         dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
 836                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_SBCNT_REG));
 837 }
 838
 839 static int
 840 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn,
 841                                struct k3_udma_glue_rx_channel_cfg *cfg)
 842 {
 843         int ret;
 844
 845         /* default rflow */
 846         if (cfg->flow_id_use_rxchan_id)
 847                 return 0;
 848
 849         /* not a GP rflows */
 850         if (rx_chn->flow_id_base != -1 &&
 851             !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
 852                 return 0;
 853
 854         /* Allocate range of GP rflows */
 855         ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax,
 856                                          rx_chn->flow_id_base,
 857                                          rx_chn->flow_num);
 858         if (ret < 0) {
 859                 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n",
 860                         rx_chn->flow_id_base, rx_chn->flow_num, ret);
 861                 return ret;
 862         }
 863         rx_chn->flow_id_base = ret;
 864
 865         return 0;
 866 }
 867
 868 static struct k3_udma_glue_rx_channel *
 869 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
 870                                  struct k3_udma_glue_rx_channel_cfg *cfg)
 871 {
 872         struct k3_udma_glue_rx_channel *rx_chn;
 873         struct psil_endpoint_config *ep_cfg;
 874         int ret, i;
 875
 876         if (cfg->flow_id_num <= 0)
 877                 return ERR_PTR(-EINVAL);
 878
 879         if (cfg->flow_id_num != 1 &&
 880             (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id))
 881                 return ERR_PTR(-EINVAL);
 882
 883         rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
 884         if (!rx_chn)
 885                 return ERR_PTR(-ENOMEM);
 886
 887         rx_chn->common.dev = dev;
 888         rx_chn->common.swdata_size = cfg->swdata_size;
 889         rx_chn->remote = false;
 890
 891         /* parse of udmap channel */
 892         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
 893                                         &rx_chn->common, false);
 894         if (ret)
 895                 goto err;
 896
 897         rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
 898                                                 rx_chn->common.psdata_size,
 899                                                 rx_chn->common.swdata_size);
 900
 901         ep_cfg = rx_chn->common.ep_config;
 902
 903         if (xudma_is_pktdma(rx_chn->common.udmax))
 904                 rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id;
 905         else
 906                 rx_chn->udma_rchan_id = -1;
 907
 908         /* request and cfg UDMAP RX channel */
 909         rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax,
 910                                               rx_chn->udma_rchan_id);
 911         if (IS_ERR(rx_chn->udma_rchanx)) {
 912                 ret = PTR_ERR(rx_chn->udma_rchanx);
 913                 dev_err(dev, "UDMAX rchanx get err %d\n", ret);
 914                 goto err;
 915         }
 916         rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
 917
 918         rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
 919         rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
 920         dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x",
 921                      rx_chn->udma_rchan_id, rx_chn->common.src_thread);
 922         ret = device_register(&rx_chn->common.chan_dev);
 923         if (ret) {
 924                 dev_err(dev, "Channel Device registration failed %d\n", ret);
 925                 put_device(&rx_chn->common.chan_dev);
 926                 rx_chn->common.chan_dev.parent = NULL;
 927                 goto err;
 928         }
 929
 930         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 931                 /* prepare the channel device as coherent */
 932                 rx_chn->common.chan_dev.dma_coherent = true;
 933                 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
 934                                              DMA_BIT_MASK(48));
 935         }
 936
 937         if (xudma_is_pktdma(rx_chn->common.udmax)) {
 938                 int flow_start = cfg->flow_id_base;
 939                 int flow_end;
 940
 941                 if (flow_start == -1)
 942                         flow_start = ep_cfg->flow_start;
 943
 944                 flow_end = flow_start + cfg->flow_id_num - 1;
 945                 if (flow_start < ep_cfg->flow_start ||
 946                     flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) {
 947                         dev_err(dev, "Invalid flow range requested\n");
 948                         ret = -EINVAL;
 949                         goto err;
 950                 }
 951                 rx_chn->flow_id_base = flow_start;
 952         } else {
 953                 rx_chn->flow_id_base = cfg->flow_id_base;
 954
 955                 /* Use RX channel id as flow id: target dev can't generate flow_id */
 956                 if (cfg->flow_id_use_rxchan_id)
 957                         rx_chn->flow_id_base = rx_chn->udma_rchan_id;
 958         }
 959
 960         rx_chn->flow_num = cfg->flow_id_num;
 961
 962         rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
 963                                      sizeof(*rx_chn->flows), GFP_KERNEL);
 964         if (!rx_chn->flows) {
 965                 ret = -ENOMEM;
 966                 goto err;
 967         }
 968
 969         ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
 970         if (ret)
 971                 goto err;
 972
 973         for (i = 0; i < rx_chn->flow_num; i++)
 974                 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
 975
 976         /* request and cfg psi-l */
 977         rx_chn->common.dst_thread =
 978                         xudma_dev_get_psil_base(rx_chn->common.udmax) +
 979                         rx_chn->udma_rchan_id;
 980
 981         ret = k3_udma_glue_cfg_rx_chn(rx_chn);
 982         if (ret) {
 983                 dev_err(dev, "Failed to cfg rchan %d\n", ret);
 984                 goto err;
 985         }
 986
 987         /* init default RX flow only if flow_num = 1 */
 988         if (cfg->def_flow_cfg) {
 989                 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg);
 990                 if (ret)
 991                         goto err;
 992         }
 993
 994         k3_udma_glue_dump_rx_chn(rx_chn);
 995
 996         return rx_chn;
 997
 998 err:
 999         k3_udma_glue_release_rx_chn(rx_chn);
1000         return ERR_PTR(ret);
1001 }
1002
1003 static struct k3_udma_glue_rx_channel *
1004 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name,
1005                                    struct k3_udma_glue_rx_channel_cfg *cfg)
1006 {
1007         struct k3_udma_glue_rx_channel *rx_chn;
1008         int ret, i;
1009
1010         if (cfg->flow_id_num <= 0 ||
1011             cfg->flow_id_use_rxchan_id ||
1012             cfg->def_flow_cfg ||
1013             cfg->flow_id_base < 0)
1014                 return ERR_PTR(-EINVAL);
1015
1016         /*
1017          * Remote RX channel is under control of Remote CPU core, so
1018          * Linux can only request and manipulate by dedicated RX flows
1019          */
1020
1021         rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
1022         if (!rx_chn)
1023                 return ERR_PTR(-ENOMEM);
1024
1025         rx_chn->common.dev = dev;
1026         rx_chn->common.swdata_size = cfg->swdata_size;
1027         rx_chn->remote = true;
1028         rx_chn->udma_rchan_id = -1;
1029         rx_chn->flow_num = cfg->flow_id_num;
1030         rx_chn->flow_id_base = cfg->flow_id_base;
1031         rx_chn->psil_paired = false;
1032
1033         /* parse of udmap channel */
1034         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
1035                                         &rx_chn->common, false);
1036         if (ret)
1037                 goto err;
1038
1039         rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
1040                                                 rx_chn->common.psdata_size,
1041                                                 rx_chn->common.swdata_size);
1042
1043         rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
1044                                      sizeof(*rx_chn->flows), GFP_KERNEL);
1045         if (!rx_chn->flows) {
1046                 ret = -ENOMEM;
1047                 goto err;
1048         }
1049
1050         rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
1051         rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
1052         dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x",
1053                      rx_chn->common.src_thread);
1054         ret = device_register(&rx_chn->common.chan_dev);
1055         if (ret) {
1056                 dev_err(dev, "Channel Device registration failed %d\n", ret);
1057                 put_device(&rx_chn->common.chan_dev);
1058                 rx_chn->common.chan_dev.parent = NULL;
1059                 goto err;
1060         }
1061
1062         if (xudma_is_pktdma(rx_chn->common.udmax)) {
1063                 /* prepare the channel device as coherent */
1064                 rx_chn->common.chan_dev.dma_coherent = true;
1065                 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
1066                                              DMA_BIT_MASK(48));
1067         }
1068
1069         ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
1070         if (ret)
1071                 goto err;
1072
1073         for (i = 0; i < rx_chn->flow_num; i++)
1074                 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
1075
1076         k3_udma_glue_dump_rx_chn(rx_chn);
1077
1078         return rx_chn;
1079
1080 err:
1081         k3_udma_glue_release_rx_chn(rx_chn);
1082         return ERR_PTR(ret);
1083 }
1084
1085 struct k3_udma_glue_rx_channel *
1086 k3_udma_glue_request_rx_chn(struct device *dev, const char *name,
1087                             struct k3_udma_glue_rx_channel_cfg *cfg)
1088 {
1089         if (cfg->remote)
1090                 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg);
1091         else
1092                 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg);
1093 }
1094 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn);
1095
1096 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1097 {
1098         int i;
1099
1100         if (IS_ERR_OR_NULL(rx_chn->common.udmax))
1101                 return;
1102
1103         if (rx_chn->psil_paired) {
1104                 xudma_navss_psil_unpair(rx_chn->common.udmax,
1105                                         rx_chn->common.src_thread,
1106                                         rx_chn->common.dst_thread);
1107                 rx_chn->psil_paired = false;
1108         }
1109
1110         for (i = 0; i < rx_chn->flow_num; i++)
1111                 k3_udma_glue_release_rx_flow(rx_chn, i);
1112
1113         if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
1114                 xudma_free_gp_rflow_range(rx_chn->common.udmax,
1115                                           rx_chn->flow_id_base,
1116                                           rx_chn->flow_num);
1117
1118         if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
1119                 xudma_rchan_put(rx_chn->common.udmax,
1120                                 rx_chn->udma_rchanx);
1121
1122         if (rx_chn->common.chan_dev.parent) {
1123                 device_unregister(&rx_chn->common.chan_dev);
1124                 rx_chn->common.chan_dev.parent = NULL;
1125         }
1126 }
1127 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn);
1128
1129 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn,
1130                               u32 flow_idx,
1131                               struct k3_udma_glue_rx_flow_cfg *flow_cfg)
1132 {
1133         if (flow_idx >= rx_chn->flow_num)
1134                 return -EINVAL;
1135
1136         return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg);
1137 }
1138 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init);
1139
1140 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn,
1141                                     u32 flow_idx)
1142 {
1143         struct k3_udma_glue_rx_flow *flow;
1144
1145         if (flow_idx >= rx_chn->flow_num)
1146                 return -EINVAL;
1147
1148         flow = &rx_chn->flows[flow_idx];
1149
1150         return k3_ringacc_get_ring_id(flow->ringrxfdq);
1151 }
1152 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id);
1153
1154 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn)
1155 {
1156         return rx_chn->flow_id_base;
1157 }
1158 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base);
1159
1160 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
1161                                 u32 flow_idx)
1162 {
1163         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1164         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1165         struct device *dev = rx_chn->common.dev;
1166         struct ti_sci_msg_rm_udmap_flow_cfg req;
1167         int rx_ring_id;
1168         int rx_ringfdq_id;
1169         int ret = 0;
1170
1171         if (!rx_chn->remote)
1172                 return -EINVAL;
1173
1174         rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
1175         rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
1176
1177         memset(&req, 0, sizeof(req));
1178
1179         req.valid_params =
1180                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1181                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1182                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1183                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1184                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1185         req.nav_id = tisci_rm->tisci_dev_id;
1186         req.flow_index = flow->udma_rflow_id;
1187         req.rx_dest_qnum = rx_ring_id;
1188         req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
1189         req.rx_fdq1_qnum = rx_ringfdq_id;
1190         req.rx_fdq2_qnum = rx_ringfdq_id;
1191         req.rx_fdq3_qnum = rx_ringfdq_id;
1192
1193         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1194         if (ret) {
1195                 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id,
1196                         ret);
1197         }
1198
1199         return ret;
1200 }
1201 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable);
1202
1203 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
1204                                  u32 flow_idx)
1205 {
1206         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1207         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1208         struct device *dev = rx_chn->common.dev;
1209         struct ti_sci_msg_rm_udmap_flow_cfg req;
1210         int ret = 0;
1211
1212         if (!rx_chn->remote)
1213                 return -EINVAL;
1214
1215         memset(&req, 0, sizeof(req));
1216         req.valid_params =
1217                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1218                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1219                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1220                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1221                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1222         req.nav_id = tisci_rm->tisci_dev_id;
1223         req.flow_index = flow->udma_rflow_id;
1224         req.rx_dest_qnum = TI_SCI_RESOURCE_NULL;
1225         req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL;
1226         req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL;
1227         req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL;
1228         req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL;
1229
1230         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1231         if (ret) {
1232                 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id,
1233                         ret);
1234         }
1235
1236         return ret;
1237 }
1238 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable);
1239
1240 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1241 {
1242         int ret;
1243
1244         if (rx_chn->remote)
1245                 return -EINVAL;
1246
1247         if (rx_chn->flows_ready < rx_chn->flow_num)
1248                 return -EINVAL;
1249
1250         ret = xudma_navss_psil_pair(rx_chn->common.udmax,
1251                                     rx_chn->common.src_thread,
1252                                     rx_chn->common.dst_thread);
1253         if (ret) {
1254                 dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret);
1255                 return ret;
1256         }
1257
1258         rx_chn->psil_paired = true;
1259
1260         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG,
1261                             UDMA_CHAN_RT_CTL_EN);
1262
1263         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1264                             UDMA_PEER_RT_EN_ENABLE);
1265
1266         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en");
1267         return 0;
1268 }
1269 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn);
1270
1271 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1272 {
1273         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1");
1274
1275         xudma_rchanrt_write(rx_chn->udma_rchanx,
1276                             UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
1277         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0);
1278
1279         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2");
1280
1281         if (rx_chn->psil_paired) {
1282                 xudma_navss_psil_unpair(rx_chn->common.udmax,
1283                                         rx_chn->common.src_thread,
1284                                         rx_chn->common.dst_thread);
1285                 rx_chn->psil_paired = false;
1286         }
1287 }
1288 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn);
1289
1290 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1291                                bool sync)
1292 {
1293         int i = 0;
1294         u32 val;
1295
1296         if (rx_chn->remote)
1297                 return;
1298
1299         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1");
1300
1301         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1302                             UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN);
1303
1304         val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG);
1305
1306         while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
1307                 val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1308                                          UDMA_CHAN_RT_CTL_REG);
1309                 udelay(1);
1310                 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
1311                         dev_err(rx_chn->common.dev, "RX tdown timeout\n");
1312                         break;
1313                 }
1314                 i++;
1315         }
1316
1317         val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1318                                  UDMA_CHAN_RT_PEER_RT_EN_REG);
1319         if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
1320                 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n");
1321         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2");
1322 }
1323 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn);
1324
1325 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1326                 u32 flow_num, void *data,
1327                 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq)
1328 {
1329         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1330         struct device *dev = rx_chn->common.dev;
1331         dma_addr_t desc_dma;
1332         int occ_rx, i, ret;
1333
1334         /* reset RXCQ as it is not input for udma - expected to be empty */
1335         occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
1336         dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
1337
1338         /* Skip RX FDQ in case one FDQ is used for the set of flows */
1339         if (skip_fdq)
1340                 goto do_reset;
1341
1342         /*
1343          * RX FDQ reset need to be special way as it is input for udma and its
1344          * state cached by udma, so:
1345          * 1) save RX FDQ occ
1346          * 2) clean up RX FDQ and call callback .cleanup() for each desc
1347          * 3) reset RX FDQ in a special way
1348          */
1349         occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq);
1350         dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx);
1351
1352         for (i = 0; i < occ_rx; i++) {
1353                 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
1354                 if (ret) {
1355                         if (ret != -ENODATA)
1356                                 dev_err(dev, "RX reset pop %d\n", ret);
1357                         break;
1358                 }
1359                 cleanup(data, desc_dma);
1360         }
1361
1362         k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
1363
1364 do_reset:
1365         k3_ringacc_ring_reset(flow->ringrx);
1366 }
1367 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn);
1368
1369 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1370                              u32 flow_num, struct cppi5_host_desc_t *desc_rx,
1371                              dma_addr_t desc_dma)
1372 {
1373         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1374
1375         return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma);
1376 }
1377 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn);
1378
1379 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1380                             u32 flow_num, dma_addr_t *desc_dma)
1381 {
1382         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1383
1384         return k3_ringacc_ring_pop(flow->ringrx, desc_dma);
1385 }
1386 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn);
1387
1388 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
1389                             u32 flow_num)
1390 {
1391         struct k3_udma_glue_rx_flow *flow;
1392
1393         flow = &rx_chn->flows[flow_num];
1394
1395         if (xudma_is_pktdma(rx_chn->common.udmax)) {
1396                 flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax,
1397                                                         flow->udma_rflow_id);
1398         } else {
1399                 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
1400         }
1401
1402         return flow->virq;
1403 }
1404 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq);
1405
1406 struct device *
1407         k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn)
1408 {
1409         if (xudma_is_pktdma(rx_chn->common.udmax) &&
1410             (rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15))
1411                 return &rx_chn->common.chan_dev;
1412
1413         return xudma_get_device(rx_chn->common.udmax);
1414 }
1415 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device);
1416
1417 void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn,
1418                                        dma_addr_t *addr)
1419 {
1420         if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1421             !rx_chn->common.atype_asel)
1422                 return;
1423
1424         *addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
1425 }
1426 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr);
1427
1428 void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn,
1429                                        dma_addr_t *addr)
1430 {
1431         if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1432             !rx_chn->common.atype_asel)
1433                 return;
1434
1435         *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1436 }
1437 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr);
1438
1439 static int __init k3_udma_glue_class_init(void)
1440 {
1441         return class_register(&k3_udma_glue_devclass);
1442 }
1443
1444 module_init(k3_udma_glue_class_init);
1445 MODULE_LICENSE("GPL v2");