drivers/net/wireless/ath/ath10k/ce.c

   1 // SPDX-License-Identifier: ISC
   2 /*
   3  * Copyright (c) 2005-2011 Atheros Communications Inc.
   4  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
   5  * Copyright (c) 2018 The Linux Foundation. All rights reserved.
   6  */
   7
   8 #include "hif.h"
   9 #include "ce.h"
  10 #include "debug.h"
  11
  12 /*
  13  * Support for Copy Engine hardware, which is mainly used for
  14  * communication between Host and Target over a PCIe interconnect.
  15  */
  16
  17 /*
  18  * A single CopyEngine (CE) comprises two "rings":
  19  *   a source ring
  20  *   a destination ring
  21  *
  22  * Each ring consists of a number of descriptors which specify
  23  * an address, length, and meta-data.
  24  *
  25  * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
  26  * controls one ring and the other side controls the other ring.
  27  * The source side chooses when to initiate a transfer and it
  28  * chooses what to send (buffer address, length). The destination
  29  * side keeps a supply of "anonymous receive buffers" available and
  30  * it handles incoming data as it arrives (when the destination
  31  * receives an interrupt).
  32  *
  33  * The sender may send a simple buffer (address/length) or it may
  34  * send a small list of buffers.  When a small list is sent, hardware
  35  * "gathers" these and they end up in a single destination buffer
  36  * with a single interrupt.
  37  *
  38  * There are several "contexts" managed by this layer -- more, it
  39  * may seem -- than should be needed. These are provided mainly for
  40  * maximum flexibility and especially to facilitate a simpler HIF
  41  * implementation. There are per-CopyEngine recv, send, and watermark
  42  * contexts. These are supplied by the caller when a recv, send,
  43  * or watermark handler is established and they are echoed back to
  44  * the caller when the respective callbacks are invoked. There is
  45  * also a per-transfer context supplied by the caller when a buffer
  46  * (or sendlist) is sent and when a buffer is enqueued for recv.
  47  * These per-transfer contexts are echoed back to the caller when
  48  * the buffer is sent/received.
  49  */
  50
  51 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
  52                                         struct ath10k_ce_pipe *ce_state)
  53 {
  54         u32 ce_id = ce_state->id;
  55         u32 addr = 0;
  56
  57         switch (ce_id) {
  58         case 0:
  59                 addr = 0x00032000;
  60                 break;
  61         case 3:
  62                 addr = 0x0003200C;
  63                 break;
  64         case 4:
  65                 addr = 0x00032010;
  66                 break;
  67         case 5:
  68                 addr = 0x00032014;
  69                 break;
  70         case 7:
  71                 addr = 0x0003201C;
  72                 break;
  73         default:
  74                 ath10k_warn(ar, "invalid CE id: %d", ce_id);
  75                 break;
  76         }
  77         return addr;
  78 }
  79
  80 static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar,
  81                                          struct ath10k_ce_pipe *ce_state)
  82 {
  83         u32 ce_id = ce_state->id;
  84         u32 addr = 0;
  85
  86         switch (ce_id) {
  87         case 1:
  88                 addr = 0x00032034;
  89                 break;
  90         case 2:
  91                 addr = 0x00032038;
  92                 break;
  93         case 5:
  94                 addr = 0x00032044;
  95                 break;
  96         case 7:
  97                 addr = 0x0003204C;
  98                 break;
  99         case 8:
 100                 addr = 0x00032050;
 101                 break;
 102         case 9:
 103                 addr = 0x00032054;
 104                 break;
 105         case 10:
 106                 addr = 0x00032058;
 107                 break;
 108         case 11:
 109                 addr = 0x0003205C;
 110                 break;
 111         default:
 112                 ath10k_warn(ar, "invalid CE id: %d", ce_id);
 113                 break;
 114         }
 115
 116         return addr;
 117 }
 118
 119 static inline unsigned int
 120 ath10k_set_ring_byte(unsigned int offset,
 121                      struct ath10k_hw_ce_regs_addr_map *addr_map)
 122 {
 123         return ((offset << addr_map->lsb) & addr_map->mask);
 124 }
 125
 126 static inline unsigned int
 127 ath10k_get_ring_byte(unsigned int offset,
 128                      struct ath10k_hw_ce_regs_addr_map *addr_map)
 129 {
 130         return ((offset & addr_map->mask) >> (addr_map->lsb));
 131 }
 132
 133 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
 134 {
 135         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 136
 137         return ce->bus_ops->read32(ar, offset);
 138 }
 139
 140 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
 141 {
 142         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 143
 144         ce->bus_ops->write32(ar, offset, value);
 145 }
 146
 147 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
 148                                                        u32 ce_ctrl_addr,
 149                                                        unsigned int n)
 150 {
 151         ath10k_ce_write32(ar, ce_ctrl_addr +
 152                           ar->hw_ce_regs->dst_wr_index_addr, n);
 153 }
 154
 155 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
 156                                                       u32 ce_ctrl_addr)
 157 {
 158         return ath10k_ce_read32(ar, ce_ctrl_addr +
 159                                 ar->hw_ce_regs->dst_wr_index_addr);
 160 }
 161
 162 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
 163                                                       u32 ce_ctrl_addr,
 164                                                       unsigned int n)
 165 {
 166         ath10k_ce_write32(ar, ce_ctrl_addr +
 167                           ar->hw_ce_regs->sr_wr_index_addr, n);
 168 }
 169
 170 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
 171                                                      u32 ce_ctrl_addr)
 172 {
 173         return ath10k_ce_read32(ar, ce_ctrl_addr +
 174                                 ar->hw_ce_regs->sr_wr_index_addr);
 175 }
 176
 177 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
 178                                                          u32 ce_id)
 179 {
 180         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 181
 182         return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
 183 }
 184
 185 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
 186                                                     u32 ce_ctrl_addr)
 187 {
 188         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 189         u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 190         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 191         u32 index;
 192
 193         if (ar->hw_params.rri_on_ddr &&
 194             (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 195                 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
 196         else
 197                 index = ath10k_ce_read32(ar, ce_ctrl_addr +
 198                                          ar->hw_ce_regs->current_srri_addr);
 199
 200         return index;
 201 }
 202
 203 static inline void
 204 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
 205                                           struct ath10k_ce_pipe *ce_state,
 206                                           unsigned int value)
 207 {
 208         ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
 209 }
 210
 211 static inline void
 212 ath10k_ce_shadow_dest_ring_write_index_set(struct ath10k *ar,
 213                                            struct ath10k_ce_pipe *ce_state,
 214                                            unsigned int value)
 215 {
 216         ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value);
 217 }
 218
 219 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
 220                                                     u32 ce_id,
 221                                                     u64 addr)
 222 {
 223         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 224         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 225         u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 226         u32 addr_lo = lower_32_bits(addr);
 227
 228         ath10k_ce_write32(ar, ce_ctrl_addr +
 229                           ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
 230
 231         if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
 232                 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
 233                                                             addr);
 234         }
 235 }
 236
 237 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
 238                                                 u32 ce_ctrl_addr,
 239                                                 u64 addr)
 240 {
 241         u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 242
 243         ath10k_ce_write32(ar, ce_ctrl_addr +
 244                           ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
 245 }
 246
 247 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
 248                                                u32 ce_ctrl_addr,
 249                                                unsigned int n)
 250 {
 251         ath10k_ce_write32(ar, ce_ctrl_addr +
 252                           ar->hw_ce_regs->sr_size_addr, n);
 253 }
 254
 255 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
 256                                                u32 ce_ctrl_addr,
 257                                                unsigned int n)
 258 {
 259         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 260
 261         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 262                                           ctrl_regs->addr);
 263
 264         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 265                           (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
 266                           ath10k_set_ring_byte(n, ctrl_regs->dmax));
 267 }
 268
 269 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
 270                                                     u32 ce_ctrl_addr,
 271                                                     unsigned int n)
 272 {
 273         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 274
 275         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 276                                           ctrl_regs->addr);
 277
 278         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 279                           (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
 280                           ath10k_set_ring_byte(n, ctrl_regs->src_ring));
 281 }
 282
 283 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
 284                                                      u32 ce_ctrl_addr,
 285                                                      unsigned int n)
 286 {
 287         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 288
 289         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 290                                           ctrl_regs->addr);
 291
 292         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 293                           (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
 294                           ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
 295 }
 296
 297 static inline
 298         u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
 299 {
 300         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 301
 302         return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
 303                 CE_DDR_RRI_MASK;
 304 }
 305
 306 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
 307                                                      u32 ce_ctrl_addr)
 308 {
 309         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 310         u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 311         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 312         u32 index;
 313
 314         if (ar->hw_params.rri_on_ddr &&
 315             (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 316                 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
 317         else
 318                 index = ath10k_ce_read32(ar, ce_ctrl_addr +
 319                                          ar->hw_ce_regs->current_drri_addr);
 320
 321         return index;
 322 }
 323
 324 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
 325                                                      u32 ce_id,
 326                                                      u64 addr)
 327 {
 328         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 329         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 330         u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 331         u32 addr_lo = lower_32_bits(addr);
 332
 333         ath10k_ce_write32(ar, ce_ctrl_addr +
 334                           ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
 335
 336         if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
 337                 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
 338                                                              addr);
 339         }
 340 }
 341
 342 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
 343                                                  u32 ce_ctrl_addr,
 344                                                  u64 addr)
 345 {
 346         u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 347         u32 reg_value;
 348
 349         reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
 350                                      ar->hw_ce_regs->dr_base_addr_hi);
 351         reg_value &= ~CE_DESC_ADDR_HI_MASK;
 352         reg_value |= addr_hi;
 353         ath10k_ce_write32(ar, ce_ctrl_addr +
 354                           ar->hw_ce_regs->dr_base_addr_hi, reg_value);
 355 }
 356
 357 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
 358                                                 u32 ce_ctrl_addr,
 359                                                 unsigned int n)
 360 {
 361         ath10k_ce_write32(ar, ce_ctrl_addr +
 362                           ar->hw_ce_regs->dr_size_addr, n);
 363 }
 364
 365 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
 366                                                    u32 ce_ctrl_addr,
 367                                                    unsigned int n)
 368 {
 369         struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 370         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 371
 372         ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 373                           (addr & ~(srcr_wm->wm_high->mask)) |
 374                           (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
 375 }
 376
 377 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
 378                                                   u32 ce_ctrl_addr,
 379                                                   unsigned int n)
 380 {
 381         struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 382         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 383
 384         ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 385                           (addr & ~(srcr_wm->wm_low->mask)) |
 386                           (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
 387 }
 388
 389 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
 390                                                     u32 ce_ctrl_addr,
 391                                                     unsigned int n)
 392 {
 393         struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 394         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 395
 396         ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 397                           (addr & ~(dstr_wm->wm_high->mask)) |
 398                           (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
 399 }
 400
 401 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
 402                                                    u32 ce_ctrl_addr,
 403                                                    unsigned int n)
 404 {
 405         struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 406         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 407
 408         ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 409                           (addr & ~(dstr_wm->wm_low->mask)) |
 410                           (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
 411 }
 412
 413 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
 414                                                         u32 ce_ctrl_addr)
 415 {
 416         struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 417
 418         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 419                                             ar->hw_ce_regs->host_ie_addr);
 420
 421         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 422                           host_ie_addr | host_ie->copy_complete->mask);
 423 }
 424
 425 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
 426                                                         u32 ce_ctrl_addr)
 427 {
 428         struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 429
 430         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 431                                             ar->hw_ce_regs->host_ie_addr);
 432
 433         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 434                           host_ie_addr & ~(host_ie->copy_complete->mask));
 435 }
 436
 437 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
 438                                                     u32 ce_ctrl_addr)
 439 {
 440         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 441
 442         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 443                                             ar->hw_ce_regs->host_ie_addr);
 444
 445         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 446                           host_ie_addr & ~(wm_regs->wm_mask));
 447 }
 448
 449 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
 450                                                u32 ce_ctrl_addr)
 451 {
 452         struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
 453
 454         u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 455                                             ar->hw_ce_regs->misc_ie_addr);
 456
 457         ath10k_ce_write32(ar,
 458                           ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
 459                           misc_ie_addr | misc_regs->err_mask);
 460 }
 461
 462 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
 463                                                 u32 ce_ctrl_addr)
 464 {
 465         struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
 466
 467         u32 misc_ie_addr = ath10k_ce_read32(ar,
 468                         ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
 469
 470         ath10k_ce_write32(ar,
 471                           ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
 472                           misc_ie_addr & ~(misc_regs->err_mask));
 473 }
 474
 475 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
 476                                                      u32 ce_ctrl_addr,
 477                                                      unsigned int mask)
 478 {
 479         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 480
 481         ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
 482 }
 483
 484 /*
 485  * Guts of ath10k_ce_send.
 486  * The caller takes responsibility for any needed locking.
 487  */
 488 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 489                                   void *per_transfer_context,
 490                                   dma_addr_t buffer,
 491                                   unsigned int nbytes,
 492                                   unsigned int transfer_id,
 493                                   unsigned int flags)
 494 {
 495         struct ath10k *ar = ce_state->ar;
 496         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 497         struct ce_desc *desc, sdesc;
 498         unsigned int nentries_mask = src_ring->nentries_mask;
 499         unsigned int sw_index = src_ring->sw_index;
 500         unsigned int write_index = src_ring->write_index;
 501         u32 ctrl_addr = ce_state->ctrl_addr;
 502         u32 desc_flags = 0;
 503         int ret = 0;
 504
 505         if (nbytes > ce_state->src_sz_max)
 506                 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 507                             __func__, nbytes, ce_state->src_sz_max);
 508
 509         if (unlikely(CE_RING_DELTA(nentries_mask,
 510                                    write_index, sw_index - 1) <= 0)) {
 511                 ret = -ENOSR;
 512                 goto exit;
 513         }
 514
 515         desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
 516                                    write_index);
 517
 518         desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 519
 520         if (flags & CE_SEND_FLAG_GATHER)
 521                 desc_flags |= CE_DESC_FLAGS_GATHER;
 522         if (flags & CE_SEND_FLAG_BYTE_SWAP)
 523                 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 524
 525         sdesc.addr   = __cpu_to_le32(buffer);
 526         sdesc.nbytes = __cpu_to_le16(nbytes);
 527         sdesc.flags  = __cpu_to_le16(desc_flags);
 528
 529         *desc = sdesc;
 530
 531         src_ring->per_transfer_context[write_index] = per_transfer_context;
 532
 533         /* Update Source Ring Write Index */
 534         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 535
 536         /* WORKAROUND */
 537         if (!(flags & CE_SEND_FLAG_GATHER))
 538                 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
 539
 540         src_ring->write_index = write_index;
 541 exit:
 542         return ret;
 543 }
 544
 545 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
 546                                      void *per_transfer_context,
 547                                      dma_addr_t buffer,
 548                                      unsigned int nbytes,
 549                                      unsigned int transfer_id,
 550                                      unsigned int flags)
 551 {
 552         struct ath10k *ar = ce_state->ar;
 553         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 554         struct ce_desc_64 *desc, sdesc;
 555         unsigned int nentries_mask = src_ring->nentries_mask;
 556         unsigned int sw_index;
 557         unsigned int write_index = src_ring->write_index;
 558         u32 ctrl_addr = ce_state->ctrl_addr;
 559         __le32 *addr;
 560         u32 desc_flags = 0;
 561         int ret = 0;
 562
 563         if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
 564                 return -ESHUTDOWN;
 565
 566         if (nbytes > ce_state->src_sz_max)
 567                 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 568                             __func__, nbytes, ce_state->src_sz_max);
 569
 570         if (ar->hw_params.rri_on_ddr)
 571                 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
 572         else
 573                 sw_index = src_ring->sw_index;
 574
 575         if (unlikely(CE_RING_DELTA(nentries_mask,
 576                                    write_index, sw_index - 1) <= 0)) {
 577                 ret = -ENOSR;
 578                 goto exit;
 579         }
 580
 581         desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
 582                                       write_index);
 583
 584         desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 585
 586         if (flags & CE_SEND_FLAG_GATHER)
 587                 desc_flags |= CE_DESC_FLAGS_GATHER;
 588
 589         if (flags & CE_SEND_FLAG_BYTE_SWAP)
 590                 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 591
 592         addr = (__le32 *)&sdesc.addr;
 593
 594         flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
 595         addr[0] = __cpu_to_le32(buffer);
 596         addr[1] = __cpu_to_le32(flags);
 597         if (flags & CE_SEND_FLAG_GATHER)
 598                 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
 599         else
 600                 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
 601
 602         sdesc.nbytes = __cpu_to_le16(nbytes);
 603         sdesc.flags  = __cpu_to_le16(desc_flags);
 604
 605         *desc = sdesc;
 606
 607         src_ring->per_transfer_context[write_index] = per_transfer_context;
 608
 609         /* Update Source Ring Write Index */
 610         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 611
 612         if (!(flags & CE_SEND_FLAG_GATHER)) {
 613                 if (ar->hw_params.shadow_reg_support)
 614                         ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
 615                                                                   write_index);
 616                 else
 617                         ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
 618                                                            write_index);
 619         }
 620
 621         src_ring->write_index = write_index;
 622 exit:
 623         return ret;
 624 }
 625
 626 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 627                           void *per_transfer_context,
 628                           dma_addr_t buffer,
 629                           unsigned int nbytes,
 630                           unsigned int transfer_id,
 631                           unsigned int flags)
 632 {
 633         return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
 634                                     buffer, nbytes, transfer_id, flags);
 635 }
 636 EXPORT_SYMBOL(ath10k_ce_send_nolock);
 637
 638 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 639 {
 640         struct ath10k *ar = pipe->ar;
 641         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 642         struct ath10k_ce_ring *src_ring = pipe->src_ring;
 643         u32 ctrl_addr = pipe->ctrl_addr;
 644
 645         lockdep_assert_held(&ce->ce_lock);
 646
 647         /*
 648          * This function must be called only if there is an incomplete
 649          * scatter-gather transfer (before index register is updated)
 650          * that needs to be cleaned up.
 651          */
 652         if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
 653                 return;
 654
 655         if (WARN_ON_ONCE(src_ring->write_index ==
 656                          ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
 657                 return;
 658
 659         src_ring->write_index--;
 660         src_ring->write_index &= src_ring->nentries_mask;
 661
 662         src_ring->per_transfer_context[src_ring->write_index] = NULL;
 663 }
 664 EXPORT_SYMBOL(__ath10k_ce_send_revert);
 665
 666 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 667                    void *per_transfer_context,
 668                    dma_addr_t buffer,
 669                    unsigned int nbytes,
 670                    unsigned int transfer_id,
 671                    unsigned int flags)
 672 {
 673         struct ath10k *ar = ce_state->ar;
 674         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 675         int ret;
 676
 677         spin_lock_bh(&ce->ce_lock);
 678         ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
 679                                     buffer, nbytes, transfer_id, flags);
 680         spin_unlock_bh(&ce->ce_lock);
 681
 682         return ret;
 683 }
 684 EXPORT_SYMBOL(ath10k_ce_send);
 685
 686 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
 687 {
 688         struct ath10k *ar = pipe->ar;
 689         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 690         int delta;
 691
 692         spin_lock_bh(&ce->ce_lock);
 693         delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
 694                               pipe->src_ring->write_index,
 695                               pipe->src_ring->sw_index - 1);
 696         spin_unlock_bh(&ce->ce_lock);
 697
 698         return delta;
 699 }
 700 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
 701
 702 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 703 {
 704         struct ath10k *ar = pipe->ar;
 705         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 706         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 707         unsigned int nentries_mask = dest_ring->nentries_mask;
 708         unsigned int write_index = dest_ring->write_index;
 709         unsigned int sw_index = dest_ring->sw_index;
 710
 711         lockdep_assert_held(&ce->ce_lock);
 712
 713         return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
 714 }
 715 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
 716
 717 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 718                                    dma_addr_t paddr)
 719 {
 720         struct ath10k *ar = pipe->ar;
 721         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 722         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 723         unsigned int nentries_mask = dest_ring->nentries_mask;
 724         unsigned int write_index = dest_ring->write_index;
 725         unsigned int sw_index = dest_ring->sw_index;
 726         struct ce_desc *base = dest_ring->base_addr_owner_space;
 727         struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
 728         u32 ctrl_addr = pipe->ctrl_addr;
 729
 730         lockdep_assert_held(&ce->ce_lock);
 731
 732         if ((pipe->id != 5) &&
 733             CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 734                 return -ENOSPC;
 735
 736         desc->addr = __cpu_to_le32(paddr);
 737         desc->nbytes = 0;
 738
 739         dest_ring->per_transfer_context[write_index] = ctx;
 740         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 741         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 742         dest_ring->write_index = write_index;
 743
 744         return 0;
 745 }
 746
 747 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
 748                                       void *ctx,
 749                                       dma_addr_t paddr)
 750 {
 751         struct ath10k *ar = pipe->ar;
 752         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 753         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 754         unsigned int nentries_mask = dest_ring->nentries_mask;
 755         unsigned int write_index = dest_ring->write_index;
 756         unsigned int sw_index = dest_ring->sw_index;
 757         struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 758         struct ce_desc_64 *desc =
 759                         CE_DEST_RING_TO_DESC_64(base, write_index);
 760         u32 ctrl_addr = pipe->ctrl_addr;
 761
 762         lockdep_assert_held(&ce->ce_lock);
 763
 764         if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 765                 return -ENOSPC;
 766
 767         desc->addr = __cpu_to_le64(paddr);
 768         desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
 769
 770         desc->nbytes = 0;
 771
 772         dest_ring->per_transfer_context[write_index] = ctx;
 773         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 774         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 775         dest_ring->write_index = write_index;
 776
 777         return 0;
 778 }
 779
 780 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 781 {
 782         struct ath10k *ar = pipe->ar;
 783         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 784         unsigned int nentries_mask = dest_ring->nentries_mask;
 785         unsigned int write_index = dest_ring->write_index;
 786         u32 ctrl_addr = pipe->ctrl_addr;
 787         u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
 788
 789         /* Prevent CE ring stuck issue that will occur when ring is full.
 790          * Make sure that write index is 1 less than read index.
 791          */
 792         if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
 793                 nentries -= 1;
 794
 795         write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
 796         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 797         dest_ring->write_index = write_index;
 798 }
 799 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
 800
 801 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 802                           dma_addr_t paddr)
 803 {
 804         struct ath10k *ar = pipe->ar;
 805         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 806         int ret;
 807
 808         spin_lock_bh(&ce->ce_lock);
 809         ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
 810         spin_unlock_bh(&ce->ce_lock);
 811
 812         return ret;
 813 }
 814 EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
 815
 816 /*
 817  * Guts of ath10k_ce_completed_recv_next.
 818  * The caller takes responsibility for any necessary locking.
 819  */
 820 static int
 821          _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 822                                                void **per_transfer_contextp,
 823                                                unsigned int *nbytesp)
 824 {
 825         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 826         unsigned int nentries_mask = dest_ring->nentries_mask;
 827         unsigned int sw_index = dest_ring->sw_index;
 828
 829         struct ce_desc *base = dest_ring->base_addr_owner_space;
 830         struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 831         struct ce_desc sdesc;
 832         u16 nbytes;
 833
 834         /* Copy in one go for performance reasons */
 835         sdesc = *desc;
 836
 837         nbytes = __le16_to_cpu(sdesc.nbytes);
 838         if (nbytes == 0) {
 839                 /*
 840                  * This closes a relatively unusual race where the Host
 841                  * sees the updated DRRI before the update to the
 842                  * corresponding descriptor has completed. We treat this
 843                  * as a descriptor that is not yet done.
 844                  */
 845                 return -EIO;
 846         }
 847
 848         desc->nbytes = 0;
 849
 850         /* Return data from completed destination descriptor */
 851         *nbytesp = nbytes;
 852
 853         if (per_transfer_contextp)
 854                 *per_transfer_contextp =
 855                         dest_ring->per_transfer_context[sw_index];
 856
 857         /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 858          * So update transfer context all CEs except CE5.
 859          */
 860         if (ce_state->id != 5)
 861                 dest_ring->per_transfer_context[sw_index] = NULL;
 862
 863         /* Update sw_index */
 864         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 865         dest_ring->sw_index = sw_index;
 866
 867         return 0;
 868 }
 869
 870 static int
 871 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
 872                                          void **per_transfer_contextp,
 873                                          unsigned int *nbytesp)
 874 {
 875         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 876         unsigned int nentries_mask = dest_ring->nentries_mask;
 877         unsigned int sw_index = dest_ring->sw_index;
 878         struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 879         struct ce_desc_64 *desc =
 880                 CE_DEST_RING_TO_DESC_64(base, sw_index);
 881         struct ce_desc_64 sdesc;
 882         u16 nbytes;
 883
 884         /* Copy in one go for performance reasons */
 885         sdesc = *desc;
 886
 887         nbytes = __le16_to_cpu(sdesc.nbytes);
 888         if (nbytes == 0) {
 889                 /* This closes a relatively unusual race where the Host
 890                  * sees the updated DRRI before the update to the
 891                  * corresponding descriptor has completed. We treat this
 892                  * as a descriptor that is not yet done.
 893                  */
 894                 return -EIO;
 895         }
 896
 897         desc->nbytes = 0;
 898
 899         /* Return data from completed destination descriptor */
 900         *nbytesp = nbytes;
 901
 902         if (per_transfer_contextp)
 903                 *per_transfer_contextp =
 904                         dest_ring->per_transfer_context[sw_index];
 905
 906         /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 907          * So update transfer context all CEs except CE5.
 908          */
 909         if (ce_state->id != 5)
 910                 dest_ring->per_transfer_context[sw_index] = NULL;
 911
 912         /* Update sw_index */
 913         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 914         dest_ring->sw_index = sw_index;
 915
 916         return 0;
 917 }
 918
 919 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 920                                          void **per_transfer_ctx,
 921                                          unsigned int *nbytesp)
 922 {
 923         return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 924                                                             per_transfer_ctx,
 925                                                             nbytesp);
 926 }
 927 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
 928
 929 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 930                                   void **per_transfer_contextp,
 931                                   unsigned int *nbytesp)
 932 {
 933         struct ath10k *ar = ce_state->ar;
 934         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 935         int ret;
 936
 937         spin_lock_bh(&ce->ce_lock);
 938         ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 939                                                    per_transfer_contextp,
 940                                                    nbytesp);
 941
 942         spin_unlock_bh(&ce->ce_lock);
 943
 944         return ret;
 945 }
 946 EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
 947
 948 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 949                                        void **per_transfer_contextp,
 950                                        dma_addr_t *bufferp)
 951 {
 952         struct ath10k_ce_ring *dest_ring;
 953         unsigned int nentries_mask;
 954         unsigned int sw_index;
 955         unsigned int write_index;
 956         int ret;
 957         struct ath10k *ar;
 958         struct ath10k_ce *ce;
 959
 960         dest_ring = ce_state->dest_ring;
 961
 962         if (!dest_ring)
 963                 return -EIO;
 964
 965         ar = ce_state->ar;
 966         ce = ath10k_ce_priv(ar);
 967
 968         spin_lock_bh(&ce->ce_lock);
 969
 970         nentries_mask = dest_ring->nentries_mask;
 971         sw_index = dest_ring->sw_index;
 972         write_index = dest_ring->write_index;
 973         if (write_index != sw_index) {
 974                 struct ce_desc *base = dest_ring->base_addr_owner_space;
 975                 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 976
 977                 /* Return data from completed destination descriptor */
 978                 *bufferp = __le32_to_cpu(desc->addr);
 979
 980                 if (per_transfer_contextp)
 981                         *per_transfer_contextp =
 982                                 dest_ring->per_transfer_context[sw_index];
 983
 984                 /* sanity */
 985                 dest_ring->per_transfer_context[sw_index] = NULL;
 986                 desc->nbytes = 0;
 987
 988                 /* Update sw_index */
 989                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 990                 dest_ring->sw_index = sw_index;
 991                 ret = 0;
 992         } else {
 993                 ret = -EIO;
 994         }
 995
 996         spin_unlock_bh(&ce->ce_lock);
 997
 998         return ret;
 999 }
1000
1001 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
1002                                           void **per_transfer_contextp,
1003                                           dma_addr_t *bufferp)
1004 {
1005         struct ath10k_ce_ring *dest_ring;
1006         unsigned int nentries_mask;
1007         unsigned int sw_index;
1008         unsigned int write_index;
1009         int ret;
1010         struct ath10k *ar;
1011         struct ath10k_ce *ce;
1012
1013         dest_ring = ce_state->dest_ring;
1014
1015         if (!dest_ring)
1016                 return -EIO;
1017
1018         ar = ce_state->ar;
1019         ce = ath10k_ce_priv(ar);
1020
1021         spin_lock_bh(&ce->ce_lock);
1022
1023         nentries_mask = dest_ring->nentries_mask;
1024         sw_index = dest_ring->sw_index;
1025         write_index = dest_ring->write_index;
1026         if (write_index != sw_index) {
1027                 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
1028                 struct ce_desc_64 *desc =
1029                         CE_DEST_RING_TO_DESC_64(base, sw_index);
1030
1031                 /* Return data from completed destination descriptor */
1032                 *bufferp = __le64_to_cpu(desc->addr);
1033
1034                 if (per_transfer_contextp)
1035                         *per_transfer_contextp =
1036                                 dest_ring->per_transfer_context[sw_index];
1037
1038                 /* sanity */
1039                 dest_ring->per_transfer_context[sw_index] = NULL;
1040                 desc->nbytes = 0;
1041
1042                 /* Update sw_index */
1043                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1044                 dest_ring->sw_index = sw_index;
1045                 ret = 0;
1046         } else {
1047                 ret = -EIO;
1048         }
1049
1050         spin_unlock_bh(&ce->ce_lock);
1051
1052         return ret;
1053 }
1054
1055 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
1056                                void **per_transfer_contextp,
1057                                dma_addr_t *bufferp)
1058 {
1059         return ce_state->ops->ce_revoke_recv_next(ce_state,
1060                                                   per_transfer_contextp,
1061                                                   bufferp);
1062 }
1063 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
1064
1065 /*
1066  * Guts of ath10k_ce_completed_send_next.
1067  * The caller takes responsibility for any necessary locking.
1068  */
1069 static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1070                                                  void **per_transfer_contextp)
1071 {
1072         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1073         u32 ctrl_addr = ce_state->ctrl_addr;
1074         struct ath10k *ar = ce_state->ar;
1075         unsigned int nentries_mask = src_ring->nentries_mask;
1076         unsigned int sw_index = src_ring->sw_index;
1077         unsigned int read_index;
1078         struct ce_desc *desc;
1079
1080         if (src_ring->hw_index == sw_index) {
1081                 /*
1082                  * The SW completion index has caught up with the cached
1083                  * version of the HW completion index.
1084                  * Update the cached HW completion index to see whether
1085                  * the SW has really caught up to the HW, or if the cached
1086                  * value of the HW index has become stale.
1087                  */
1088
1089                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1090                 if (read_index == 0xffffffff)
1091                         return -ENODEV;
1092
1093                 read_index &= nentries_mask;
1094                 src_ring->hw_index = read_index;
1095         }
1096
1097         if (ar->hw_params.rri_on_ddr)
1098                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1099         else
1100                 read_index = src_ring->hw_index;
1101
1102         if (read_index == sw_index)
1103                 return -EIO;
1104
1105         if (per_transfer_contextp)
1106                 *per_transfer_contextp =
1107                         src_ring->per_transfer_context[sw_index];
1108
1109         /* sanity */
1110         src_ring->per_transfer_context[sw_index] = NULL;
1111         desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1112                                    sw_index);
1113         desc->nbytes = 0;
1114
1115         /* Update sw_index */
1116         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1117         src_ring->sw_index = sw_index;
1118
1119         return 0;
1120 }
1121
1122 static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1123                                                     void **per_transfer_contextp)
1124 {
1125         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1126         u32 ctrl_addr = ce_state->ctrl_addr;
1127         struct ath10k *ar = ce_state->ar;
1128         unsigned int nentries_mask = src_ring->nentries_mask;
1129         unsigned int sw_index = src_ring->sw_index;
1130         unsigned int read_index;
1131         struct ce_desc_64 *desc;
1132
1133         if (src_ring->hw_index == sw_index) {
1134                 /*
1135                  * The SW completion index has caught up with the cached
1136                  * version of the HW completion index.
1137                  * Update the cached HW completion index to see whether
1138                  * the SW has really caught up to the HW, or if the cached
1139                  * value of the HW index has become stale.
1140                  */
1141
1142                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1143                 if (read_index == 0xffffffff)
1144                         return -ENODEV;
1145
1146                 read_index &= nentries_mask;
1147                 src_ring->hw_index = read_index;
1148         }
1149
1150         if (ar->hw_params.rri_on_ddr)
1151                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1152         else
1153                 read_index = src_ring->hw_index;
1154
1155         if (read_index == sw_index)
1156                 return -EIO;
1157
1158         if (per_transfer_contextp)
1159                 *per_transfer_contextp =
1160                         src_ring->per_transfer_context[sw_index];
1161
1162         /* sanity */
1163         src_ring->per_transfer_context[sw_index] = NULL;
1164         desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1165                                       sw_index);
1166         desc->nbytes = 0;
1167
1168         /* Update sw_index */
1169         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1170         src_ring->sw_index = sw_index;
1171
1172         return 0;
1173 }
1174
1175 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1176                                          void **per_transfer_contextp)
1177 {
1178         return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1179                                                             per_transfer_contextp);
1180 }
1181 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1182
1183 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1184                                         struct ath10k_ce_ring *src_ring,
1185                                         u32 sw_index,
1186                                         dma_addr_t *bufferp,
1187                                         u32 *nbytesp,
1188                                         u32 *transfer_idp)
1189 {
1190                 struct ce_desc *base = src_ring->base_addr_owner_space;
1191                 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1192
1193                 /* Return data from completed source descriptor */
1194                 *bufferp = __le32_to_cpu(desc->addr);
1195                 *nbytesp = __le16_to_cpu(desc->nbytes);
1196                 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1197                                    CE_DESC_FLAGS_META_DATA);
1198 }
1199
1200 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1201                                            struct ath10k_ce_ring *src_ring,
1202                                            u32 sw_index,
1203                                            dma_addr_t *bufferp,
1204                                            u32 *nbytesp,
1205                                            u32 *transfer_idp)
1206 {
1207                 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1208                 struct ce_desc_64 *desc =
1209                         CE_SRC_RING_TO_DESC_64(base, sw_index);
1210
1211                 /* Return data from completed source descriptor */
1212                 *bufferp = __le64_to_cpu(desc->addr);
1213                 *nbytesp = __le16_to_cpu(desc->nbytes);
1214                 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1215                                    CE_DESC_FLAGS_META_DATA);
1216 }
1217
1218 /* NB: Modeled after ath10k_ce_completed_send_next */
1219 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1220                                void **per_transfer_contextp,
1221                                dma_addr_t *bufferp,
1222                                unsigned int *nbytesp,
1223                                unsigned int *transfer_idp)
1224 {
1225         struct ath10k_ce_ring *src_ring;
1226         unsigned int nentries_mask;
1227         unsigned int sw_index;
1228         unsigned int write_index;
1229         int ret;
1230         struct ath10k *ar;
1231         struct ath10k_ce *ce;
1232
1233         src_ring = ce_state->src_ring;
1234
1235         if (!src_ring)
1236                 return -EIO;
1237
1238         ar = ce_state->ar;
1239         ce = ath10k_ce_priv(ar);
1240
1241         spin_lock_bh(&ce->ce_lock);
1242
1243         nentries_mask = src_ring->nentries_mask;
1244         sw_index = src_ring->sw_index;
1245         write_index = src_ring->write_index;
1246
1247         if (write_index != sw_index) {
1248                 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1249                                                     bufferp, nbytesp,
1250                                                     transfer_idp);
1251
1252                 if (per_transfer_contextp)
1253                         *per_transfer_contextp =
1254                                 src_ring->per_transfer_context[sw_index];
1255
1256                 /* sanity */
1257                 src_ring->per_transfer_context[sw_index] = NULL;
1258
1259                 /* Update sw_index */
1260                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1261                 src_ring->sw_index = sw_index;
1262                 ret = 0;
1263         } else {
1264                 ret = -EIO;
1265         }
1266
1267         spin_unlock_bh(&ce->ce_lock);
1268
1269         return ret;
1270 }
1271 EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1272
1273 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1274                                   void **per_transfer_contextp)
1275 {
1276         struct ath10k *ar = ce_state->ar;
1277         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1278         int ret;
1279
1280         spin_lock_bh(&ce->ce_lock);
1281         ret = ath10k_ce_completed_send_next_nolock(ce_state,
1282                                                    per_transfer_contextp);
1283         spin_unlock_bh(&ce->ce_lock);
1284
1285         return ret;
1286 }
1287 EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1288
1289 /*
1290  * Guts of interrupt handler for per-engine interrupts on a particular CE.
1291  *
1292  * Invokes registered callbacks for recv_complete,
1293  * send_complete, and watermarks.
1294  */
1295 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1296 {
1297         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1298         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1299         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1300         u32 ctrl_addr = ce_state->ctrl_addr;
1301
1302         /*
1303          * Clear before handling
1304          *
1305          * Misc CE interrupts are not being handled, but still need
1306          * to be cleared.
1307          *
1308          * NOTE: When the last copy engine interrupt is cleared the
1309          * hardware will go to sleep.  Once this happens any access to
1310          * the CE registers can cause a hardware fault.
1311          */
1312         ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1313                                           wm_regs->cc_mask | wm_regs->wm_mask);
1314
1315         if (ce_state->recv_cb)
1316                 ce_state->recv_cb(ce_state);
1317
1318         if (ce_state->send_cb)
1319                 ce_state->send_cb(ce_state);
1320 }
1321 EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1322
1323 /*
1324  * Handler for per-engine interrupts on ALL active CEs.
1325  * This is used in cases where the system is sharing a
1326  * single interrput for all CEs
1327  */
1328
1329 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1330 {
1331         int ce_id;
1332         u32 intr_summary;
1333
1334         intr_summary = ath10k_ce_interrupt_summary(ar);
1335
1336         for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1337                 if (intr_summary & (1 << ce_id))
1338                         intr_summary &= ~(1 << ce_id);
1339                 else
1340                         /* no intr pending on this CE */
1341                         continue;
1342
1343                 ath10k_ce_per_engine_service(ar, ce_id);
1344         }
1345 }
1346 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1347
1348 /*
1349  * Adjust interrupts for the copy complete handler.
1350  * If it's needed for either send or recv, then unmask
1351  * this interrupt; otherwise, mask it.
1352  *
1353  * Called with ce_lock held.
1354  */
1355 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1356 {
1357         u32 ctrl_addr = ce_state->ctrl_addr;
1358         struct ath10k *ar = ce_state->ar;
1359         bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1360
1361         if ((!disable_copy_compl_intr) &&
1362             (ce_state->send_cb || ce_state->recv_cb))
1363                 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1364         else
1365                 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1366
1367         ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1368 }
1369
1370 void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)
1371 {
1372         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1373         struct ath10k_ce_pipe *ce_state;
1374         u32 ctrl_addr;
1375
1376         ce_state  = &ce->ce_states[ce_id];
1377         if (ce_state->attr_flags & CE_ATTR_POLL)
1378                 return;
1379
1380         ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1381
1382         ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1383         ath10k_ce_error_intr_disable(ar, ctrl_addr);
1384         ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1385 }
1386 EXPORT_SYMBOL(ath10k_ce_disable_interrupt);
1387
1388 void ath10k_ce_disable_interrupts(struct ath10k *ar)
1389 {
1390         int ce_id;
1391
1392         for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1393                 ath10k_ce_disable_interrupt(ar, ce_id);
1394 }
1395 EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1396
1397 void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)
1398 {
1399         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1400         struct ath10k_ce_pipe *ce_state;
1401
1402         ce_state  = &ce->ce_states[ce_id];
1403         if (ce_state->attr_flags & CE_ATTR_POLL)
1404                 return;
1405
1406         ath10k_ce_per_engine_handler_adjust(ce_state);
1407 }
1408 EXPORT_SYMBOL(ath10k_ce_enable_interrupt);
1409
1410 void ath10k_ce_enable_interrupts(struct ath10k *ar)
1411 {
1412         int ce_id;
1413
1414         /* Enable interrupts for copy engine that
1415          * are not using polling mode.
1416          */
1417         for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1418                 ath10k_ce_enable_interrupt(ar, ce_id);
1419 }
1420 EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1421
1422 static int ath10k_ce_init_src_ring(struct ath10k *ar,
1423                                    unsigned int ce_id,
1424                                    const struct ce_attr *attr)
1425 {
1426         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1427         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1428         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1429         u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1430
1431         nentries = roundup_pow_of_two(attr->src_nentries);
1432
1433         if (ar->hw_params.target_64bit)
1434                 memset(src_ring->base_addr_owner_space, 0,
1435                        nentries * sizeof(struct ce_desc_64));
1436         else
1437                 memset(src_ring->base_addr_owner_space, 0,
1438                        nentries * sizeof(struct ce_desc));
1439
1440         src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1441         src_ring->sw_index &= src_ring->nentries_mask;
1442         src_ring->hw_index = src_ring->sw_index;
1443
1444         src_ring->write_index =
1445                 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1446         src_ring->write_index &= src_ring->nentries_mask;
1447
1448         ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1449                                          src_ring->base_addr_ce_space);
1450         ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1451         ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1452         ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1453         ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1454         ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1455
1456         ath10k_dbg(ar, ATH10K_DBG_BOOT,
1457                    "boot init ce src ring id %d entries %d base_addr %pK\n",
1458                    ce_id, nentries, src_ring->base_addr_owner_space);
1459
1460         return 0;
1461 }
1462
1463 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1464                                     unsigned int ce_id,
1465                                     const struct ce_attr *attr)
1466 {
1467         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1468         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1469         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1470         u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1471
1472         nentries = roundup_pow_of_two(attr->dest_nentries);
1473
1474         if (ar->hw_params.target_64bit)
1475                 memset(dest_ring->base_addr_owner_space, 0,
1476                        nentries * sizeof(struct ce_desc_64));
1477         else
1478                 memset(dest_ring->base_addr_owner_space, 0,
1479                        nentries * sizeof(struct ce_desc));
1480
1481         dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1482         dest_ring->sw_index &= dest_ring->nentries_mask;
1483         dest_ring->write_index =
1484                 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1485         dest_ring->write_index &= dest_ring->nentries_mask;
1486
1487         ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1488                                           dest_ring->base_addr_ce_space);
1489         ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1490         ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1491         ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1492         ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1493
1494         ath10k_dbg(ar, ATH10K_DBG_BOOT,
1495                    "boot ce dest ring id %d entries %d base_addr %pK\n",
1496                    ce_id, nentries, dest_ring->base_addr_owner_space);
1497
1498         return 0;
1499 }
1500
1501 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1502                                        struct ath10k_ce_ring *src_ring,
1503                                        u32 nentries)
1504 {
1505         src_ring->shadow_base_unaligned = kcalloc(nentries,
1506                                                   sizeof(struct ce_desc_64),
1507                                                   GFP_KERNEL);
1508         if (!src_ring->shadow_base_unaligned)
1509                 return -ENOMEM;
1510
1511         src_ring->shadow_base = (struct ce_desc_64 *)
1512                         PTR_ALIGN(src_ring->shadow_base_unaligned,
1513                                   CE_DESC_RING_ALIGN);
1514         return 0;
1515 }
1516
1517 static struct ath10k_ce_ring *
1518 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1519                          const struct ce_attr *attr)
1520 {
1521         struct ath10k_ce_ring *src_ring;
1522         u32 nentries = attr->src_nentries;
1523         dma_addr_t base_addr;
1524         int ret;
1525
1526         nentries = roundup_pow_of_two(nentries);
1527
1528         src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1529                                        nentries), GFP_KERNEL);
1530         if (src_ring == NULL)
1531                 return ERR_PTR(-ENOMEM);
1532
1533         src_ring->nentries = nentries;
1534         src_ring->nentries_mask = nentries - 1;
1535
1536         /*
1537          * Legacy platforms that do not support cache
1538          * coherent DMA are unsupported
1539          */
1540         src_ring->base_addr_owner_space_unaligned =
1541                 dma_alloc_coherent(ar->dev,
1542                                    (nentries * sizeof(struct ce_desc) +
1543                                     CE_DESC_RING_ALIGN),
1544                                    &base_addr, GFP_KERNEL);
1545         if (!src_ring->base_addr_owner_space_unaligned) {
1546                 kfree(src_ring);
1547                 return ERR_PTR(-ENOMEM);
1548         }
1549
1550         src_ring->base_addr_ce_space_unaligned = base_addr;
1551
1552         src_ring->base_addr_owner_space =
1553                         PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1554                                   CE_DESC_RING_ALIGN);
1555         src_ring->base_addr_ce_space =
1556                         ALIGN(src_ring->base_addr_ce_space_unaligned,
1557                               CE_DESC_RING_ALIGN);
1558
1559         if (ar->hw_params.shadow_reg_support) {
1560                 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1561                 if (ret) {
1562                         dma_free_coherent(ar->dev,
1563                                           (nentries * sizeof(struct ce_desc) +
1564                                            CE_DESC_RING_ALIGN),
1565                                           src_ring->base_addr_owner_space_unaligned,
1566                                           base_addr);
1567                         kfree(src_ring);
1568                         return ERR_PTR(ret);
1569                 }
1570         }
1571
1572         return src_ring;
1573 }
1574
1575 static struct ath10k_ce_ring *
1576 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1577                             const struct ce_attr *attr)
1578 {
1579         struct ath10k_ce_ring *src_ring;
1580         u32 nentries = attr->src_nentries;
1581         dma_addr_t base_addr;
1582         int ret;
1583
1584         nentries = roundup_pow_of_two(nentries);
1585
1586         src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1587                                        nentries), GFP_KERNEL);
1588         if (!src_ring)
1589                 return ERR_PTR(-ENOMEM);
1590
1591         src_ring->nentries = nentries;
1592         src_ring->nentries_mask = nentries - 1;
1593
1594         /* Legacy platforms that do not support cache
1595          * coherent DMA are unsupported
1596          */
1597         src_ring->base_addr_owner_space_unaligned =
1598                 dma_alloc_coherent(ar->dev,
1599                                    (nentries * sizeof(struct ce_desc_64) +
1600                                     CE_DESC_RING_ALIGN),
1601                                    &base_addr, GFP_KERNEL);
1602         if (!src_ring->base_addr_owner_space_unaligned) {
1603                 kfree(src_ring);
1604                 return ERR_PTR(-ENOMEM);
1605         }
1606
1607         src_ring->base_addr_ce_space_unaligned = base_addr;
1608
1609         src_ring->base_addr_owner_space =
1610                         PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1611                                   CE_DESC_RING_ALIGN);
1612         src_ring->base_addr_ce_space =
1613                         ALIGN(src_ring->base_addr_ce_space_unaligned,
1614                               CE_DESC_RING_ALIGN);
1615
1616         if (ar->hw_params.shadow_reg_support) {
1617                 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1618                 if (ret) {
1619                         dma_free_coherent(ar->dev,
1620                                           (nentries * sizeof(struct ce_desc_64) +
1621                                            CE_DESC_RING_ALIGN),
1622                                           src_ring->base_addr_owner_space_unaligned,
1623                                           base_addr);
1624                         kfree(src_ring);
1625                         return ERR_PTR(ret);
1626                 }
1627         }
1628
1629         return src_ring;
1630 }
1631
1632 static struct ath10k_ce_ring *
1633 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1634                           const struct ce_attr *attr)
1635 {
1636         struct ath10k_ce_ring *dest_ring;
1637         u32 nentries;
1638         dma_addr_t base_addr;
1639
1640         nentries = roundup_pow_of_two(attr->dest_nentries);
1641
1642         dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1643                                         nentries), GFP_KERNEL);
1644         if (dest_ring == NULL)
1645                 return ERR_PTR(-ENOMEM);
1646
1647         dest_ring->nentries = nentries;
1648         dest_ring->nentries_mask = nentries - 1;
1649
1650         /*
1651          * Legacy platforms that do not support cache
1652          * coherent DMA are unsupported
1653          */
1654         dest_ring->base_addr_owner_space_unaligned =
1655                 dma_alloc_coherent(ar->dev,
1656                                    (nentries * sizeof(struct ce_desc) +
1657                                     CE_DESC_RING_ALIGN),
1658                                    &base_addr, GFP_KERNEL);
1659         if (!dest_ring->base_addr_owner_space_unaligned) {
1660                 kfree(dest_ring);
1661                 return ERR_PTR(-ENOMEM);
1662         }
1663
1664         dest_ring->base_addr_ce_space_unaligned = base_addr;
1665
1666         dest_ring->base_addr_owner_space =
1667                         PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1668                                   CE_DESC_RING_ALIGN);
1669         dest_ring->base_addr_ce_space =
1670                                 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1671                                       CE_DESC_RING_ALIGN);
1672
1673         return dest_ring;
1674 }
1675
1676 static struct ath10k_ce_ring *
1677 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1678                              const struct ce_attr *attr)
1679 {
1680         struct ath10k_ce_ring *dest_ring;
1681         u32 nentries;
1682         dma_addr_t base_addr;
1683
1684         nentries = roundup_pow_of_two(attr->dest_nentries);
1685
1686         dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1687                                         nentries), GFP_KERNEL);
1688         if (!dest_ring)
1689                 return ERR_PTR(-ENOMEM);
1690
1691         dest_ring->nentries = nentries;
1692         dest_ring->nentries_mask = nentries - 1;
1693
1694         /* Legacy platforms that do not support cache
1695          * coherent DMA are unsupported
1696          */
1697         dest_ring->base_addr_owner_space_unaligned =
1698                 dma_alloc_coherent(ar->dev,
1699                                    (nentries * sizeof(struct ce_desc_64) +
1700                                     CE_DESC_RING_ALIGN),
1701                                    &base_addr, GFP_KERNEL);
1702         if (!dest_ring->base_addr_owner_space_unaligned) {
1703                 kfree(dest_ring);
1704                 return ERR_PTR(-ENOMEM);
1705         }
1706
1707         dest_ring->base_addr_ce_space_unaligned = base_addr;
1708
1709         /* Correctly initialize memory to 0 to prevent garbage
1710          * data crashing system when download firmware
1711          */
1712         dest_ring->base_addr_owner_space =
1713                         PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1714                                   CE_DESC_RING_ALIGN);
1715         dest_ring->base_addr_ce_space =
1716                         ALIGN(dest_ring->base_addr_ce_space_unaligned,
1717                               CE_DESC_RING_ALIGN);
1718
1719         return dest_ring;
1720 }
1721
1722 /*
1723  * Initialize a Copy Engine based on caller-supplied attributes.
1724  * This may be called once to initialize both source and destination
1725  * rings or it may be called twice for separate source and destination
1726  * initialization. It may be that only one side or the other is
1727  * initialized by software/firmware.
1728  */
1729 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1730                         const struct ce_attr *attr)
1731 {
1732         int ret;
1733
1734         if (attr->src_nentries) {
1735                 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1736                 if (ret) {
1737                         ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1738                                    ce_id, ret);
1739                         return ret;
1740                 }
1741         }
1742
1743         if (attr->dest_nentries) {
1744                 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1745                 if (ret) {
1746                         ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1747                                    ce_id, ret);
1748                         return ret;
1749                 }
1750         }
1751
1752         return 0;
1753 }
1754 EXPORT_SYMBOL(ath10k_ce_init_pipe);
1755
1756 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1757 {
1758         u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1759
1760         ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1761         ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1762         ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1763         ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1764 }
1765
1766 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1767 {
1768         u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1769
1770         ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1771         ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1772         ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1773 }
1774
1775 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1776 {
1777         ath10k_ce_deinit_src_ring(ar, ce_id);
1778         ath10k_ce_deinit_dest_ring(ar, ce_id);
1779 }
1780 EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1781
1782 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1783 {
1784         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1785         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1786
1787         if (ce_state->src_ring) {
1788                 if (ar->hw_params.shadow_reg_support)
1789                         kfree(ce_state->src_ring->shadow_base_unaligned);
1790                 dma_free_coherent(ar->dev,
1791                                   (ce_state->src_ring->nentries *
1792                                    sizeof(struct ce_desc) +
1793                                    CE_DESC_RING_ALIGN),
1794                                   ce_state->src_ring->base_addr_owner_space,
1795                                   ce_state->src_ring->base_addr_ce_space);
1796                 kfree(ce_state->src_ring);
1797         }
1798
1799         if (ce_state->dest_ring) {
1800                 dma_free_coherent(ar->dev,
1801                                   (ce_state->dest_ring->nentries *
1802                                    sizeof(struct ce_desc) +
1803                                    CE_DESC_RING_ALIGN),
1804                                   ce_state->dest_ring->base_addr_owner_space,
1805                                   ce_state->dest_ring->base_addr_ce_space);
1806                 kfree(ce_state->dest_ring);
1807         }
1808
1809         ce_state->src_ring = NULL;
1810         ce_state->dest_ring = NULL;
1811 }
1812
1813 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1814 {
1815         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1816         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1817
1818         if (ce_state->src_ring) {
1819                 if (ar->hw_params.shadow_reg_support)
1820                         kfree(ce_state->src_ring->shadow_base_unaligned);
1821                 dma_free_coherent(ar->dev,
1822                                   (ce_state->src_ring->nentries *
1823                                    sizeof(struct ce_desc_64) +
1824                                    CE_DESC_RING_ALIGN),
1825                                   ce_state->src_ring->base_addr_owner_space,
1826                                   ce_state->src_ring->base_addr_ce_space);
1827                 kfree(ce_state->src_ring);
1828         }
1829
1830         if (ce_state->dest_ring) {
1831                 dma_free_coherent(ar->dev,
1832                                   (ce_state->dest_ring->nentries *
1833                                    sizeof(struct ce_desc_64) +
1834                                    CE_DESC_RING_ALIGN),
1835                                   ce_state->dest_ring->base_addr_owner_space,
1836                                   ce_state->dest_ring->base_addr_ce_space);
1837                 kfree(ce_state->dest_ring);
1838         }
1839
1840         ce_state->src_ring = NULL;
1841         ce_state->dest_ring = NULL;
1842 }
1843
1844 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1845 {
1846         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1847         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1848
1849         ce_state->ops->ce_free_pipe(ar, ce_id);
1850 }
1851 EXPORT_SYMBOL(ath10k_ce_free_pipe);
1852
1853 void ath10k_ce_dump_registers(struct ath10k *ar,
1854                               struct ath10k_fw_crash_data *crash_data)
1855 {
1856         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1857         struct ath10k_ce_crash_data ce_data;
1858         u32 addr, id;
1859
1860         lockdep_assert_held(&ar->dump_mutex);
1861
1862         ath10k_err(ar, "Copy Engine register dump:\n");
1863
1864         spin_lock_bh(&ce->ce_lock);
1865         for (id = 0; id < CE_COUNT; id++) {
1866                 addr = ath10k_ce_base_address(ar, id);
1867                 ce_data.base_addr = cpu_to_le32(addr);
1868
1869                 ce_data.src_wr_idx =
1870                         cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1871                 ce_data.src_r_idx =
1872                         cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1873                 ce_data.dst_wr_idx =
1874                         cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1875                 ce_data.dst_r_idx =
1876                         cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1877
1878                 if (crash_data)
1879                         crash_data->ce_crash_data[id] = ce_data;
1880
1881                 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1882                            le32_to_cpu(ce_data.base_addr),
1883                            le32_to_cpu(ce_data.src_wr_idx),
1884                            le32_to_cpu(ce_data.src_r_idx),
1885                            le32_to_cpu(ce_data.dst_wr_idx),
1886                            le32_to_cpu(ce_data.dst_r_idx));
1887         }
1888
1889         spin_unlock_bh(&ce->ce_lock);
1890 }
1891 EXPORT_SYMBOL(ath10k_ce_dump_registers);
1892
1893 static const struct ath10k_ce_ops ce_ops = {
1894         .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1895         .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1896         .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1897         .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1898         .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1899         .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1900         .ce_free_pipe = _ath10k_ce_free_pipe,
1901         .ce_send_nolock = _ath10k_ce_send_nolock,
1902         .ce_set_src_ring_base_addr_hi = NULL,
1903         .ce_set_dest_ring_base_addr_hi = NULL,
1904         .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1905 };
1906
1907 static const struct ath10k_ce_ops ce_64_ops = {
1908         .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1909         .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1910         .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1911         .ce_completed_recv_next_nolock =
1912                                 _ath10k_ce_completed_recv_next_nolock_64,
1913         .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1914         .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1915         .ce_free_pipe = _ath10k_ce_free_pipe_64,
1916         .ce_send_nolock = _ath10k_ce_send_nolock_64,
1917         .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1918         .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1919         .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1920 };
1921
1922 static void ath10k_ce_set_ops(struct ath10k *ar,
1923                               struct ath10k_ce_pipe *ce_state)
1924 {
1925         switch (ar->hw_rev) {
1926         case ATH10K_HW_WCN3990:
1927                 ce_state->ops = &ce_64_ops;
1928                 break;
1929         default:
1930                 ce_state->ops = &ce_ops;
1931                 break;
1932         }
1933 }
1934
1935 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1936                          const struct ce_attr *attr)
1937 {
1938         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1939         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1940         int ret;
1941
1942         ath10k_ce_set_ops(ar, ce_state);
1943         /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1944          * additional TX locking checks.
1945          *
1946          * For the lack of a better place do the check here.
1947          */
1948         BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1949                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1950         BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1951                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1952         BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1953                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1954
1955         ce_state->ar = ar;
1956         ce_state->id = ce_id;
1957         ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1958         ce_state->attr_flags = attr->flags;
1959         ce_state->src_sz_max = attr->src_sz_max;
1960
1961         if (attr->src_nentries)
1962                 ce_state->send_cb = attr->send_cb;
1963
1964         if (attr->dest_nentries)
1965                 ce_state->recv_cb = attr->recv_cb;
1966
1967         if (attr->src_nentries) {
1968                 ce_state->src_ring =
1969                         ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1970                 if (IS_ERR(ce_state->src_ring)) {
1971                         ret = PTR_ERR(ce_state->src_ring);
1972                         ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1973                                    ce_id, ret);
1974                         ce_state->src_ring = NULL;
1975                         return ret;
1976                 }
1977         }
1978
1979         if (attr->dest_nentries) {
1980                 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1981                                                                         ce_id,
1982                                                                         attr);
1983                 if (IS_ERR(ce_state->dest_ring)) {
1984                         ret = PTR_ERR(ce_state->dest_ring);
1985                         ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1986                                    ce_id, ret);
1987                         ce_state->dest_ring = NULL;
1988                         return ret;
1989                 }
1990         }
1991
1992         return 0;
1993 }
1994 EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1995
1996 void ath10k_ce_alloc_rri(struct ath10k *ar)
1997 {
1998         int i;
1999         u32 value;
2000         u32 ctrl1_regs;
2001         u32 ce_base_addr;
2002         struct ath10k_ce *ce = ath10k_ce_priv(ar);
2003
2004         ce->vaddr_rri = dma_alloc_coherent(ar->dev,
2005                                            (CE_COUNT * sizeof(u32)),
2006                                            &ce->paddr_rri, GFP_KERNEL);
2007
2008         if (!ce->vaddr_rri)
2009                 return;
2010
2011         ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
2012                           lower_32_bits(ce->paddr_rri));
2013         ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
2014                           (upper_32_bits(ce->paddr_rri) &
2015                           CE_DESC_ADDR_HI_MASK));
2016
2017         for (i = 0; i < CE_COUNT; i++) {
2018                 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
2019                 ce_base_addr = ath10k_ce_base_address(ar, i);
2020                 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
2021                 value |= ar->hw_ce_regs->upd->mask;
2022                 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
2023         }
2024 }
2025 EXPORT_SYMBOL(ath10k_ce_alloc_rri);
2026
2027 void ath10k_ce_free_rri(struct ath10k *ar)
2028 {
2029         struct ath10k_ce *ce = ath10k_ce_priv(ar);
2030
2031         dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
2032                           ce->vaddr_rri,
2033                           ce->paddr_rri);
2034 }
2035 EXPORT_SYMBOL(ath10k_ce_free_rri);