drivers/net/wireless/ath/ath10k/ce.c

   1 // SPDX-License-Identifier: ISC
   2 /*
   3  * Copyright (c) 2005-2011 Atheros Communications Inc.
   4  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
   5  * Copyright (c) 2018 The Linux Foundation. All rights reserved.
   6  * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
   7  */
   8
   9 #include "hif.h"
  10 #include "ce.h"
  11 #include "debug.h"
  12
  13 /*
  14  * Support for Copy Engine hardware, which is mainly used for
  15  * communication between Host and Target over a PCIe interconnect.
  16  */
  17
  18 /*
  19  * A single CopyEngine (CE) comprises two "rings":
  20  *   a source ring
  21  *   a destination ring
  22  *
  23  * Each ring consists of a number of descriptors which specify
  24  * an address, length, and meta-data.
  25  *
  26  * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
  27  * controls one ring and the other side controls the other ring.
  28  * The source side chooses when to initiate a transfer and it
  29  * chooses what to send (buffer address, length). The destination
  30  * side keeps a supply of "anonymous receive buffers" available and
  31  * it handles incoming data as it arrives (when the destination
  32  * receives an interrupt).
  33  *
  34  * The sender may send a simple buffer (address/length) or it may
  35  * send a small list of buffers.  When a small list is sent, hardware
  36  * "gathers" these and they end up in a single destination buffer
  37  * with a single interrupt.
  38  *
  39  * There are several "contexts" managed by this layer -- more, it
  40  * may seem -- than should be needed. These are provided mainly for
  41  * maximum flexibility and especially to facilitate a simpler HIF
  42  * implementation. There are per-CopyEngine recv, send, and watermark
  43  * contexts. These are supplied by the caller when a recv, send,
  44  * or watermark handler is established and they are echoed back to
  45  * the caller when the respective callbacks are invoked. There is
  46  * also a per-transfer context supplied by the caller when a buffer
  47  * (or sendlist) is sent and when a buffer is enqueued for recv.
  48  * These per-transfer contexts are echoed back to the caller when
  49  * the buffer is sent/received.
  50  */
  51
  52 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
  53                                         struct ath10k_ce_pipe *ce_state)
  54 {
  55         u32 ce_id = ce_state->id;
  56         u32 addr = 0;
  57
  58         switch (ce_id) {
  59         case 0:
  60                 addr = 0x00032000;
  61                 break;
  62         case 3:
  63                 addr = 0x0003200C;
  64                 break;
  65         case 4:
  66                 addr = 0x00032010;
  67                 break;
  68         case 5:
  69                 addr = 0x00032014;
  70                 break;
  71         case 7:
  72                 addr = 0x0003201C;
  73                 break;
  74         default:
  75                 ath10k_warn(ar, "invalid CE id: %d", ce_id);
  76                 break;
  77         }
  78         return addr;
  79 }
  80
  81 static inline unsigned int
  82 ath10k_set_ring_byte(unsigned int offset,
  83                      struct ath10k_hw_ce_regs_addr_map *addr_map)
  84 {
  85         return ((offset << addr_map->lsb) & addr_map->mask);
  86 }
  87
  88 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
  89 {
  90         struct ath10k_ce *ce = ath10k_ce_priv(ar);
  91
  92         return ce->bus_ops->read32(ar, offset);
  93 }
  94
  95 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
  96 {
  97         struct ath10k_ce *ce = ath10k_ce_priv(ar);
  98
  99         ce->bus_ops->write32(ar, offset, value);
 100 }
 101
 102 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
 103                                                        u32 ce_ctrl_addr,
 104                                                        unsigned int n)
 105 {
 106         ath10k_ce_write32(ar, ce_ctrl_addr +
 107                           ar->hw_ce_regs->dst_wr_index_addr, n);
 108 }
 109
 110 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
 111                                                       u32 ce_ctrl_addr)
 112 {
 113         return ath10k_ce_read32(ar, ce_ctrl_addr +
 114                                 ar->hw_ce_regs->dst_wr_index_addr);
 115 }
 116
 117 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
 118                                                       u32 ce_ctrl_addr,
 119                                                       unsigned int n)
 120 {
 121         ath10k_ce_write32(ar, ce_ctrl_addr +
 122                           ar->hw_ce_regs->sr_wr_index_addr, n);
 123 }
 124
 125 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
 126                                                      u32 ce_ctrl_addr)
 127 {
 128         return ath10k_ce_read32(ar, ce_ctrl_addr +
 129                                 ar->hw_ce_regs->sr_wr_index_addr);
 130 }
 131
 132 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
 133                                                          u32 ce_id)
 134 {
 135         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 136
 137         return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
 138 }
 139
 140 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
 141                                                     u32 ce_ctrl_addr)
 142 {
 143         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 144         u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 145         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 146         u32 index;
 147
 148         if (ar->hw_params.rri_on_ddr &&
 149             (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 150                 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
 151         else
 152                 index = ath10k_ce_read32(ar, ce_ctrl_addr +
 153                                          ar->hw_ce_regs->current_srri_addr);
 154
 155         return index;
 156 }
 157
 158 static inline void
 159 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
 160                                           struct ath10k_ce_pipe *ce_state,
 161                                           unsigned int value)
 162 {
 163         ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
 164 }
 165
 166 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
 167                                                     u32 ce_id,
 168                                                     u64 addr)
 169 {
 170         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 171         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 172         u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 173         u32 addr_lo = lower_32_bits(addr);
 174
 175         ath10k_ce_write32(ar, ce_ctrl_addr +
 176                           ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
 177
 178         if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
 179                 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
 180                                                             addr);
 181         }
 182 }
 183
 184 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
 185                                                 u32 ce_ctrl_addr,
 186                                                 u64 addr)
 187 {
 188         u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 189
 190         ath10k_ce_write32(ar, ce_ctrl_addr +
 191                           ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
 192 }
 193
 194 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
 195                                                u32 ce_ctrl_addr,
 196                                                unsigned int n)
 197 {
 198         ath10k_ce_write32(ar, ce_ctrl_addr +
 199                           ar->hw_ce_regs->sr_size_addr, n);
 200 }
 201
 202 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
 203                                                u32 ce_ctrl_addr,
 204                                                unsigned int n)
 205 {
 206         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 207
 208         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 209                                           ctrl_regs->addr);
 210
 211         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 212                           (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
 213                           ath10k_set_ring_byte(n, ctrl_regs->dmax));
 214 }
 215
 216 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
 217                                                     u32 ce_ctrl_addr,
 218                                                     unsigned int n)
 219 {
 220         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 221
 222         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 223                                           ctrl_regs->addr);
 224
 225         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 226                           (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
 227                           ath10k_set_ring_byte(n, ctrl_regs->src_ring));
 228 }
 229
 230 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
 231                                                      u32 ce_ctrl_addr,
 232                                                      unsigned int n)
 233 {
 234         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 235
 236         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 237                                           ctrl_regs->addr);
 238
 239         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 240                           (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
 241                           ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
 242 }
 243
 244 static inline
 245         u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
 246 {
 247         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 248
 249         return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
 250                 CE_DDR_RRI_MASK;
 251 }
 252
 253 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
 254                                                      u32 ce_ctrl_addr)
 255 {
 256         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 257         u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 258         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 259         u32 index;
 260
 261         if (ar->hw_params.rri_on_ddr &&
 262             (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 263                 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
 264         else
 265                 index = ath10k_ce_read32(ar, ce_ctrl_addr +
 266                                          ar->hw_ce_regs->current_drri_addr);
 267
 268         return index;
 269 }
 270
 271 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
 272                                                      u32 ce_id,
 273                                                      u64 addr)
 274 {
 275         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 276         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 277         u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 278         u32 addr_lo = lower_32_bits(addr);
 279
 280         ath10k_ce_write32(ar, ce_ctrl_addr +
 281                           ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
 282
 283         if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
 284                 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
 285                                                              addr);
 286         }
 287 }
 288
 289 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
 290                                                  u32 ce_ctrl_addr,
 291                                                  u64 addr)
 292 {
 293         u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 294         u32 reg_value;
 295
 296         reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
 297                                      ar->hw_ce_regs->dr_base_addr_hi);
 298         reg_value &= ~CE_DESC_ADDR_HI_MASK;
 299         reg_value |= addr_hi;
 300         ath10k_ce_write32(ar, ce_ctrl_addr +
 301                           ar->hw_ce_regs->dr_base_addr_hi, reg_value);
 302 }
 303
 304 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
 305                                                 u32 ce_ctrl_addr,
 306                                                 unsigned int n)
 307 {
 308         ath10k_ce_write32(ar, ce_ctrl_addr +
 309                           ar->hw_ce_regs->dr_size_addr, n);
 310 }
 311
 312 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
 313                                                    u32 ce_ctrl_addr,
 314                                                    unsigned int n)
 315 {
 316         struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 317         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 318
 319         ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 320                           (addr & ~(srcr_wm->wm_high->mask)) |
 321                           (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
 322 }
 323
 324 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
 325                                                   u32 ce_ctrl_addr,
 326                                                   unsigned int n)
 327 {
 328         struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 329         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 330
 331         ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 332                           (addr & ~(srcr_wm->wm_low->mask)) |
 333                           (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
 334 }
 335
 336 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
 337                                                     u32 ce_ctrl_addr,
 338                                                     unsigned int n)
 339 {
 340         struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 341         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 342
 343         ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 344                           (addr & ~(dstr_wm->wm_high->mask)) |
 345                           (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
 346 }
 347
 348 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
 349                                                    u32 ce_ctrl_addr,
 350                                                    unsigned int n)
 351 {
 352         struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 353         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 354
 355         ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 356                           (addr & ~(dstr_wm->wm_low->mask)) |
 357                           (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
 358 }
 359
 360 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
 361                                                         u32 ce_ctrl_addr)
 362 {
 363         struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 364
 365         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 366                                             ar->hw_ce_regs->host_ie_addr);
 367
 368         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 369                           host_ie_addr | host_ie->copy_complete->mask);
 370 }
 371
 372 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
 373                                                         u32 ce_ctrl_addr)
 374 {
 375         struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 376
 377         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 378                                             ar->hw_ce_regs->host_ie_addr);
 379
 380         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 381                           host_ie_addr & ~(host_ie->copy_complete->mask));
 382 }
 383
 384 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
 385                                                     u32 ce_ctrl_addr)
 386 {
 387         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 388
 389         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 390                                             ar->hw_ce_regs->host_ie_addr);
 391
 392         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 393                           host_ie_addr & ~(wm_regs->wm_mask));
 394 }
 395
 396 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
 397                                                 u32 ce_ctrl_addr)
 398 {
 399         struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
 400
 401         u32 misc_ie_addr = ath10k_ce_read32(ar,
 402                         ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
 403
 404         ath10k_ce_write32(ar,
 405                           ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
 406                           misc_ie_addr & ~(misc_regs->err_mask));
 407 }
 408
 409 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
 410                                                      u32 ce_ctrl_addr,
 411                                                      unsigned int mask)
 412 {
 413         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 414
 415         ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
 416 }
 417
 418 /*
 419  * Guts of ath10k_ce_send.
 420  * The caller takes responsibility for any needed locking.
 421  */
 422 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 423                                   void *per_transfer_context,
 424                                   dma_addr_t buffer,
 425                                   unsigned int nbytes,
 426                                   unsigned int transfer_id,
 427                                   unsigned int flags)
 428 {
 429         struct ath10k *ar = ce_state->ar;
 430         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 431         struct ce_desc *desc, sdesc;
 432         unsigned int nentries_mask = src_ring->nentries_mask;
 433         unsigned int sw_index = src_ring->sw_index;
 434         unsigned int write_index = src_ring->write_index;
 435         u32 ctrl_addr = ce_state->ctrl_addr;
 436         u32 desc_flags = 0;
 437         int ret = 0;
 438
 439         if (nbytes > ce_state->src_sz_max)
 440                 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 441                             __func__, nbytes, ce_state->src_sz_max);
 442
 443         if (unlikely(CE_RING_DELTA(nentries_mask,
 444                                    write_index, sw_index - 1) <= 0)) {
 445                 ret = -ENOSR;
 446                 goto exit;
 447         }
 448
 449         desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
 450                                    write_index);
 451
 452         desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 453
 454         if (flags & CE_SEND_FLAG_GATHER)
 455                 desc_flags |= CE_DESC_FLAGS_GATHER;
 456         if (flags & CE_SEND_FLAG_BYTE_SWAP)
 457                 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 458
 459         sdesc.addr   = __cpu_to_le32(buffer);
 460         sdesc.nbytes = __cpu_to_le16(nbytes);
 461         sdesc.flags  = __cpu_to_le16(desc_flags);
 462
 463         *desc = sdesc;
 464
 465         src_ring->per_transfer_context[write_index] = per_transfer_context;
 466
 467         /* Update Source Ring Write Index */
 468         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 469
 470         /* WORKAROUND */
 471         if (!(flags & CE_SEND_FLAG_GATHER))
 472                 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
 473
 474         src_ring->write_index = write_index;
 475 exit:
 476         return ret;
 477 }
 478
 479 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
 480                                      void *per_transfer_context,
 481                                      dma_addr_t buffer,
 482                                      unsigned int nbytes,
 483                                      unsigned int transfer_id,
 484                                      unsigned int flags)
 485 {
 486         struct ath10k *ar = ce_state->ar;
 487         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 488         struct ce_desc_64 *desc, sdesc;
 489         unsigned int nentries_mask = src_ring->nentries_mask;
 490         unsigned int sw_index;
 491         unsigned int write_index = src_ring->write_index;
 492         u32 ctrl_addr = ce_state->ctrl_addr;
 493         __le32 *addr;
 494         u32 desc_flags = 0;
 495         int ret = 0;
 496
 497         if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
 498                 return -ESHUTDOWN;
 499
 500         if (nbytes > ce_state->src_sz_max)
 501                 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 502                             __func__, nbytes, ce_state->src_sz_max);
 503
 504         if (ar->hw_params.rri_on_ddr)
 505                 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
 506         else
 507                 sw_index = src_ring->sw_index;
 508
 509         if (unlikely(CE_RING_DELTA(nentries_mask,
 510                                    write_index, sw_index - 1) <= 0)) {
 511                 ret = -ENOSR;
 512                 goto exit;
 513         }
 514
 515         desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
 516                                       write_index);
 517
 518         desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 519
 520         if (flags & CE_SEND_FLAG_GATHER)
 521                 desc_flags |= CE_DESC_FLAGS_GATHER;
 522
 523         if (flags & CE_SEND_FLAG_BYTE_SWAP)
 524                 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 525
 526         addr = (__le32 *)&sdesc.addr;
 527
 528         flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
 529         addr[0] = __cpu_to_le32(buffer);
 530         addr[1] = __cpu_to_le32(flags);
 531         if (flags & CE_SEND_FLAG_GATHER)
 532                 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
 533         else
 534                 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
 535
 536         sdesc.nbytes = __cpu_to_le16(nbytes);
 537         sdesc.flags  = __cpu_to_le16(desc_flags);
 538
 539         *desc = sdesc;
 540
 541         src_ring->per_transfer_context[write_index] = per_transfer_context;
 542
 543         /* Update Source Ring Write Index */
 544         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 545
 546         if (!(flags & CE_SEND_FLAG_GATHER)) {
 547                 if (ar->hw_params.shadow_reg_support)
 548                         ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
 549                                                                   write_index);
 550                 else
 551                         ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
 552                                                            write_index);
 553         }
 554
 555         src_ring->write_index = write_index;
 556 exit:
 557         return ret;
 558 }
 559
 560 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 561                           void *per_transfer_context,
 562                           dma_addr_t buffer,
 563                           unsigned int nbytes,
 564                           unsigned int transfer_id,
 565                           unsigned int flags)
 566 {
 567         return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
 568                                     buffer, nbytes, transfer_id, flags);
 569 }
 570 EXPORT_SYMBOL(ath10k_ce_send_nolock);
 571
 572 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 573 {
 574         struct ath10k *ar = pipe->ar;
 575         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 576         struct ath10k_ce_ring *src_ring = pipe->src_ring;
 577         u32 ctrl_addr = pipe->ctrl_addr;
 578
 579         lockdep_assert_held(&ce->ce_lock);
 580
 581         /*
 582          * This function must be called only if there is an incomplete
 583          * scatter-gather transfer (before index register is updated)
 584          * that needs to be cleaned up.
 585          */
 586         if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
 587                 return;
 588
 589         if (WARN_ON_ONCE(src_ring->write_index ==
 590                          ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
 591                 return;
 592
 593         src_ring->write_index--;
 594         src_ring->write_index &= src_ring->nentries_mask;
 595
 596         src_ring->per_transfer_context[src_ring->write_index] = NULL;
 597 }
 598 EXPORT_SYMBOL(__ath10k_ce_send_revert);
 599
 600 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 601                    void *per_transfer_context,
 602                    dma_addr_t buffer,
 603                    unsigned int nbytes,
 604                    unsigned int transfer_id,
 605                    unsigned int flags)
 606 {
 607         struct ath10k *ar = ce_state->ar;
 608         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 609         int ret;
 610
 611         spin_lock_bh(&ce->ce_lock);
 612         ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
 613                                     buffer, nbytes, transfer_id, flags);
 614         spin_unlock_bh(&ce->ce_lock);
 615
 616         return ret;
 617 }
 618 EXPORT_SYMBOL(ath10k_ce_send);
 619
 620 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
 621 {
 622         struct ath10k *ar = pipe->ar;
 623         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 624         int delta;
 625
 626         spin_lock_bh(&ce->ce_lock);
 627         delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
 628                               pipe->src_ring->write_index,
 629                               pipe->src_ring->sw_index - 1);
 630         spin_unlock_bh(&ce->ce_lock);
 631
 632         return delta;
 633 }
 634 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
 635
 636 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 637 {
 638         struct ath10k *ar = pipe->ar;
 639         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 640         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 641         unsigned int nentries_mask = dest_ring->nentries_mask;
 642         unsigned int write_index = dest_ring->write_index;
 643         unsigned int sw_index = dest_ring->sw_index;
 644
 645         lockdep_assert_held(&ce->ce_lock);
 646
 647         return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
 648 }
 649 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
 650
 651 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 652                                    dma_addr_t paddr)
 653 {
 654         struct ath10k *ar = pipe->ar;
 655         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 656         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 657         unsigned int nentries_mask = dest_ring->nentries_mask;
 658         unsigned int write_index = dest_ring->write_index;
 659         unsigned int sw_index = dest_ring->sw_index;
 660         struct ce_desc *base = dest_ring->base_addr_owner_space;
 661         struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
 662         u32 ctrl_addr = pipe->ctrl_addr;
 663
 664         lockdep_assert_held(&ce->ce_lock);
 665
 666         if ((pipe->id != 5) &&
 667             CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 668                 return -ENOSPC;
 669
 670         desc->addr = __cpu_to_le32(paddr);
 671         desc->nbytes = 0;
 672
 673         dest_ring->per_transfer_context[write_index] = ctx;
 674         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 675         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 676         dest_ring->write_index = write_index;
 677
 678         return 0;
 679 }
 680
 681 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
 682                                       void *ctx,
 683                                       dma_addr_t paddr)
 684 {
 685         struct ath10k *ar = pipe->ar;
 686         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 687         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 688         unsigned int nentries_mask = dest_ring->nentries_mask;
 689         unsigned int write_index = dest_ring->write_index;
 690         unsigned int sw_index = dest_ring->sw_index;
 691         struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 692         struct ce_desc_64 *desc =
 693                         CE_DEST_RING_TO_DESC_64(base, write_index);
 694         u32 ctrl_addr = pipe->ctrl_addr;
 695
 696         lockdep_assert_held(&ce->ce_lock);
 697
 698         if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 699                 return -ENOSPC;
 700
 701         desc->addr = __cpu_to_le64(paddr);
 702         desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
 703
 704         desc->nbytes = 0;
 705
 706         dest_ring->per_transfer_context[write_index] = ctx;
 707         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 708         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 709         dest_ring->write_index = write_index;
 710
 711         return 0;
 712 }
 713
 714 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 715 {
 716         struct ath10k *ar = pipe->ar;
 717         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 718         unsigned int nentries_mask = dest_ring->nentries_mask;
 719         unsigned int write_index = dest_ring->write_index;
 720         u32 ctrl_addr = pipe->ctrl_addr;
 721         u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
 722
 723         /* Prevent CE ring stuck issue that will occur when ring is full.
 724          * Make sure that write index is 1 less than read index.
 725          */
 726         if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
 727                 nentries -= 1;
 728
 729         write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
 730         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 731         dest_ring->write_index = write_index;
 732 }
 733 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
 734
 735 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 736                           dma_addr_t paddr)
 737 {
 738         struct ath10k *ar = pipe->ar;
 739         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 740         int ret;
 741
 742         spin_lock_bh(&ce->ce_lock);
 743         ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
 744         spin_unlock_bh(&ce->ce_lock);
 745
 746         return ret;
 747 }
 748 EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
 749
 750 /*
 751  * Guts of ath10k_ce_completed_recv_next.
 752  * The caller takes responsibility for any necessary locking.
 753  */
 754 static int
 755          _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 756                                                void **per_transfer_contextp,
 757                                                unsigned int *nbytesp)
 758 {
 759         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 760         unsigned int nentries_mask = dest_ring->nentries_mask;
 761         unsigned int sw_index = dest_ring->sw_index;
 762
 763         struct ce_desc *base = dest_ring->base_addr_owner_space;
 764         struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 765         struct ce_desc sdesc;
 766         u16 nbytes;
 767
 768         /* Copy in one go for performance reasons */
 769         sdesc = *desc;
 770
 771         nbytes = __le16_to_cpu(sdesc.nbytes);
 772         if (nbytes == 0) {
 773                 /*
 774                  * This closes a relatively unusual race where the Host
 775                  * sees the updated DRRI before the update to the
 776                  * corresponding descriptor has completed. We treat this
 777                  * as a descriptor that is not yet done.
 778                  */
 779                 return -EIO;
 780         }
 781
 782         desc->nbytes = 0;
 783
 784         /* Return data from completed destination descriptor */
 785         *nbytesp = nbytes;
 786
 787         if (per_transfer_contextp)
 788                 *per_transfer_contextp =
 789                         dest_ring->per_transfer_context[sw_index];
 790
 791         /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 792          * So update transfer context all CEs except CE5.
 793          */
 794         if (ce_state->id != 5)
 795                 dest_ring->per_transfer_context[sw_index] = NULL;
 796
 797         /* Update sw_index */
 798         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 799         dest_ring->sw_index = sw_index;
 800
 801         return 0;
 802 }
 803
 804 static int
 805 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
 806                                          void **per_transfer_contextp,
 807                                          unsigned int *nbytesp)
 808 {
 809         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 810         unsigned int nentries_mask = dest_ring->nentries_mask;
 811         unsigned int sw_index = dest_ring->sw_index;
 812         struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 813         struct ce_desc_64 *desc =
 814                 CE_DEST_RING_TO_DESC_64(base, sw_index);
 815         struct ce_desc_64 sdesc;
 816         u16 nbytes;
 817
 818         /* Copy in one go for performance reasons */
 819         sdesc = *desc;
 820
 821         nbytes = __le16_to_cpu(sdesc.nbytes);
 822         if (nbytes == 0) {
 823                 /* This closes a relatively unusual race where the Host
 824                  * sees the updated DRRI before the update to the
 825                  * corresponding descriptor has completed. We treat this
 826                  * as a descriptor that is not yet done.
 827                  */
 828                 return -EIO;
 829         }
 830
 831         desc->nbytes = 0;
 832
 833         /* Return data from completed destination descriptor */
 834         *nbytesp = nbytes;
 835
 836         if (per_transfer_contextp)
 837                 *per_transfer_contextp =
 838                         dest_ring->per_transfer_context[sw_index];
 839
 840         /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 841          * So update transfer context all CEs except CE5.
 842          */
 843         if (ce_state->id != 5)
 844                 dest_ring->per_transfer_context[sw_index] = NULL;
 845
 846         /* Update sw_index */
 847         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 848         dest_ring->sw_index = sw_index;
 849
 850         return 0;
 851 }
 852
 853 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 854                                          void **per_transfer_ctx,
 855                                          unsigned int *nbytesp)
 856 {
 857         return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 858                                                             per_transfer_ctx,
 859                                                             nbytesp);
 860 }
 861 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
 862
 863 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 864                                   void **per_transfer_contextp,
 865                                   unsigned int *nbytesp)
 866 {
 867         struct ath10k *ar = ce_state->ar;
 868         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 869         int ret;
 870
 871         spin_lock_bh(&ce->ce_lock);
 872         ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 873                                                    per_transfer_contextp,
 874                                                    nbytesp);
 875
 876         spin_unlock_bh(&ce->ce_lock);
 877
 878         return ret;
 879 }
 880 EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
 881
 882 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 883                                        void **per_transfer_contextp,
 884                                        dma_addr_t *bufferp)
 885 {
 886         struct ath10k_ce_ring *dest_ring;
 887         unsigned int nentries_mask;
 888         unsigned int sw_index;
 889         unsigned int write_index;
 890         int ret;
 891         struct ath10k *ar;
 892         struct ath10k_ce *ce;
 893
 894         dest_ring = ce_state->dest_ring;
 895
 896         if (!dest_ring)
 897                 return -EIO;
 898
 899         ar = ce_state->ar;
 900         ce = ath10k_ce_priv(ar);
 901
 902         spin_lock_bh(&ce->ce_lock);
 903
 904         nentries_mask = dest_ring->nentries_mask;
 905         sw_index = dest_ring->sw_index;
 906         write_index = dest_ring->write_index;
 907         if (write_index != sw_index) {
 908                 struct ce_desc *base = dest_ring->base_addr_owner_space;
 909                 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 910
 911                 /* Return data from completed destination descriptor */
 912                 *bufferp = __le32_to_cpu(desc->addr);
 913
 914                 if (per_transfer_contextp)
 915                         *per_transfer_contextp =
 916                                 dest_ring->per_transfer_context[sw_index];
 917
 918                 /* sanity */
 919                 dest_ring->per_transfer_context[sw_index] = NULL;
 920                 desc->nbytes = 0;
 921
 922                 /* Update sw_index */
 923                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 924                 dest_ring->sw_index = sw_index;
 925                 ret = 0;
 926         } else {
 927                 ret = -EIO;
 928         }
 929
 930         spin_unlock_bh(&ce->ce_lock);
 931
 932         return ret;
 933 }
 934
 935 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
 936                                           void **per_transfer_contextp,
 937                                           dma_addr_t *bufferp)
 938 {
 939         struct ath10k_ce_ring *dest_ring;
 940         unsigned int nentries_mask;
 941         unsigned int sw_index;
 942         unsigned int write_index;
 943         int ret;
 944         struct ath10k *ar;
 945         struct ath10k_ce *ce;
 946
 947         dest_ring = ce_state->dest_ring;
 948
 949         if (!dest_ring)
 950                 return -EIO;
 951
 952         ar = ce_state->ar;
 953         ce = ath10k_ce_priv(ar);
 954
 955         spin_lock_bh(&ce->ce_lock);
 956
 957         nentries_mask = dest_ring->nentries_mask;
 958         sw_index = dest_ring->sw_index;
 959         write_index = dest_ring->write_index;
 960         if (write_index != sw_index) {
 961                 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 962                 struct ce_desc_64 *desc =
 963                         CE_DEST_RING_TO_DESC_64(base, sw_index);
 964
 965                 /* Return data from completed destination descriptor */
 966                 *bufferp = __le64_to_cpu(desc->addr);
 967
 968                 if (per_transfer_contextp)
 969                         *per_transfer_contextp =
 970                                 dest_ring->per_transfer_context[sw_index];
 971
 972                 /* sanity */
 973                 dest_ring->per_transfer_context[sw_index] = NULL;
 974                 desc->nbytes = 0;
 975
 976                 /* Update sw_index */
 977                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 978                 dest_ring->sw_index = sw_index;
 979                 ret = 0;
 980         } else {
 981                 ret = -EIO;
 982         }
 983
 984         spin_unlock_bh(&ce->ce_lock);
 985
 986         return ret;
 987 }
 988
 989 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 990                                void **per_transfer_contextp,
 991                                dma_addr_t *bufferp)
 992 {
 993         return ce_state->ops->ce_revoke_recv_next(ce_state,
 994                                                   per_transfer_contextp,
 995                                                   bufferp);
 996 }
 997 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
 998
 999 /*
1000  * Guts of ath10k_ce_completed_send_next.
1001  * The caller takes responsibility for any necessary locking.
1002  */
1003 static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1004                                                  void **per_transfer_contextp)
1005 {
1006         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1007         u32 ctrl_addr = ce_state->ctrl_addr;
1008         struct ath10k *ar = ce_state->ar;
1009         unsigned int nentries_mask = src_ring->nentries_mask;
1010         unsigned int sw_index = src_ring->sw_index;
1011         unsigned int read_index;
1012         struct ce_desc *desc;
1013
1014         if (src_ring->hw_index == sw_index) {
1015                 /*
1016                  * The SW completion index has caught up with the cached
1017                  * version of the HW completion index.
1018                  * Update the cached HW completion index to see whether
1019                  * the SW has really caught up to the HW, or if the cached
1020                  * value of the HW index has become stale.
1021                  */
1022
1023                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1024                 if (read_index == 0xffffffff)
1025                         return -ENODEV;
1026
1027                 read_index &= nentries_mask;
1028                 src_ring->hw_index = read_index;
1029         }
1030
1031         if (ar->hw_params.rri_on_ddr)
1032                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1033         else
1034                 read_index = src_ring->hw_index;
1035
1036         if (read_index == sw_index)
1037                 return -EIO;
1038
1039         if (per_transfer_contextp)
1040                 *per_transfer_contextp =
1041                         src_ring->per_transfer_context[sw_index];
1042
1043         /* sanity */
1044         src_ring->per_transfer_context[sw_index] = NULL;
1045         desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1046                                    sw_index);
1047         desc->nbytes = 0;
1048
1049         /* Update sw_index */
1050         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1051         src_ring->sw_index = sw_index;
1052
1053         return 0;
1054 }
1055
1056 static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1057                                                     void **per_transfer_contextp)
1058 {
1059         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1060         u32 ctrl_addr = ce_state->ctrl_addr;
1061         struct ath10k *ar = ce_state->ar;
1062         unsigned int nentries_mask = src_ring->nentries_mask;
1063         unsigned int sw_index = src_ring->sw_index;
1064         unsigned int read_index;
1065         struct ce_desc_64 *desc;
1066
1067         if (src_ring->hw_index == sw_index) {
1068                 /*
1069                  * The SW completion index has caught up with the cached
1070                  * version of the HW completion index.
1071                  * Update the cached HW completion index to see whether
1072                  * the SW has really caught up to the HW, or if the cached
1073                  * value of the HW index has become stale.
1074                  */
1075
1076                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1077                 if (read_index == 0xffffffff)
1078                         return -ENODEV;
1079
1080                 read_index &= nentries_mask;
1081                 src_ring->hw_index = read_index;
1082         }
1083
1084         if (ar->hw_params.rri_on_ddr)
1085                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1086         else
1087                 read_index = src_ring->hw_index;
1088
1089         if (read_index == sw_index)
1090                 return -EIO;
1091
1092         if (per_transfer_contextp)
1093                 *per_transfer_contextp =
1094                         src_ring->per_transfer_context[sw_index];
1095
1096         /* sanity */
1097         src_ring->per_transfer_context[sw_index] = NULL;
1098         desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1099                                       sw_index);
1100         desc->nbytes = 0;
1101
1102         /* Update sw_index */
1103         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1104         src_ring->sw_index = sw_index;
1105
1106         return 0;
1107 }
1108
1109 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1110                                          void **per_transfer_contextp)
1111 {
1112         return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1113                                                             per_transfer_contextp);
1114 }
1115 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1116
1117 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1118                                         struct ath10k_ce_ring *src_ring,
1119                                         u32 sw_index,
1120                                         dma_addr_t *bufferp,
1121                                         u32 *nbytesp,
1122                                         u32 *transfer_idp)
1123 {
1124                 struct ce_desc *base = src_ring->base_addr_owner_space;
1125                 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1126
1127                 /* Return data from completed source descriptor */
1128                 *bufferp = __le32_to_cpu(desc->addr);
1129                 *nbytesp = __le16_to_cpu(desc->nbytes);
1130                 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1131                                    CE_DESC_FLAGS_META_DATA);
1132 }
1133
1134 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1135                                            struct ath10k_ce_ring *src_ring,
1136                                            u32 sw_index,
1137                                            dma_addr_t *bufferp,
1138                                            u32 *nbytesp,
1139                                            u32 *transfer_idp)
1140 {
1141                 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1142                 struct ce_desc_64 *desc =
1143                         CE_SRC_RING_TO_DESC_64(base, sw_index);
1144
1145                 /* Return data from completed source descriptor */
1146                 *bufferp = __le64_to_cpu(desc->addr);
1147                 *nbytesp = __le16_to_cpu(desc->nbytes);
1148                 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1149                                    CE_DESC_FLAGS_META_DATA);
1150 }
1151
1152 /* NB: Modeled after ath10k_ce_completed_send_next */
1153 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1154                                void **per_transfer_contextp,
1155                                dma_addr_t *bufferp,
1156                                unsigned int *nbytesp,
1157                                unsigned int *transfer_idp)
1158 {
1159         struct ath10k_ce_ring *src_ring;
1160         unsigned int nentries_mask;
1161         unsigned int sw_index;
1162         unsigned int write_index;
1163         int ret;
1164         struct ath10k *ar;
1165         struct ath10k_ce *ce;
1166
1167         src_ring = ce_state->src_ring;
1168
1169         if (!src_ring)
1170                 return -EIO;
1171
1172         ar = ce_state->ar;
1173         ce = ath10k_ce_priv(ar);
1174
1175         spin_lock_bh(&ce->ce_lock);
1176
1177         nentries_mask = src_ring->nentries_mask;
1178         sw_index = src_ring->sw_index;
1179         write_index = src_ring->write_index;
1180
1181         if (write_index != sw_index) {
1182                 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1183                                                     bufferp, nbytesp,
1184                                                     transfer_idp);
1185
1186                 if (per_transfer_contextp)
1187                         *per_transfer_contextp =
1188                                 src_ring->per_transfer_context[sw_index];
1189
1190                 /* sanity */
1191                 src_ring->per_transfer_context[sw_index] = NULL;
1192
1193                 /* Update sw_index */
1194                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1195                 src_ring->sw_index = sw_index;
1196                 ret = 0;
1197         } else {
1198                 ret = -EIO;
1199         }
1200
1201         spin_unlock_bh(&ce->ce_lock);
1202
1203         return ret;
1204 }
1205 EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1206
1207 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1208                                   void **per_transfer_contextp)
1209 {
1210         struct ath10k *ar = ce_state->ar;
1211         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1212         int ret;
1213
1214         spin_lock_bh(&ce->ce_lock);
1215         ret = ath10k_ce_completed_send_next_nolock(ce_state,
1216                                                    per_transfer_contextp);
1217         spin_unlock_bh(&ce->ce_lock);
1218
1219         return ret;
1220 }
1221 EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1222
1223 /*
1224  * Guts of interrupt handler for per-engine interrupts on a particular CE.
1225  *
1226  * Invokes registered callbacks for recv_complete,
1227  * send_complete, and watermarks.
1228  */
1229 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1230 {
1231         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1232         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1233         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1234         u32 ctrl_addr = ce_state->ctrl_addr;
1235
1236         /*
1237          * Clear before handling
1238          *
1239          * Misc CE interrupts are not being handled, but still need
1240          * to be cleared.
1241          *
1242          * NOTE: When the last copy engine interrupt is cleared the
1243          * hardware will go to sleep.  Once this happens any access to
1244          * the CE registers can cause a hardware fault.
1245          */
1246         ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1247                                           wm_regs->cc_mask | wm_regs->wm_mask);
1248
1249         if (ce_state->recv_cb)
1250                 ce_state->recv_cb(ce_state);
1251
1252         if (ce_state->send_cb)
1253                 ce_state->send_cb(ce_state);
1254 }
1255 EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1256
1257 /*
1258  * Handler for per-engine interrupts on ALL active CEs.
1259  * This is used in cases where the system is sharing a
1260  * single interrupt for all CEs
1261  */
1262
1263 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1264 {
1265         int ce_id;
1266         u32 intr_summary;
1267
1268         intr_summary = ath10k_ce_interrupt_summary(ar);
1269
1270         for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1271                 if (intr_summary & (1 << ce_id))
1272                         intr_summary &= ~(1 << ce_id);
1273                 else
1274                         /* no intr pending on this CE */
1275                         continue;
1276
1277                 ath10k_ce_per_engine_service(ar, ce_id);
1278         }
1279 }
1280 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1281
1282 /*
1283  * Adjust interrupts for the copy complete handler.
1284  * If it's needed for either send or recv, then unmask
1285  * this interrupt; otherwise, mask it.
1286  *
1287  * Called with ce_lock held.
1288  */
1289 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1290 {
1291         u32 ctrl_addr = ce_state->ctrl_addr;
1292         struct ath10k *ar = ce_state->ar;
1293         bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1294
1295         if ((!disable_copy_compl_intr) &&
1296             (ce_state->send_cb || ce_state->recv_cb))
1297                 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1298         else
1299                 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1300
1301         ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1302 }
1303
1304 void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)
1305 {
1306         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1307         struct ath10k_ce_pipe *ce_state;
1308         u32 ctrl_addr;
1309
1310         ce_state  = &ce->ce_states[ce_id];
1311         if (ce_state->attr_flags & CE_ATTR_POLL)
1312                 return;
1313
1314         ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1315
1316         ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1317         ath10k_ce_error_intr_disable(ar, ctrl_addr);
1318         ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1319 }
1320 EXPORT_SYMBOL(ath10k_ce_disable_interrupt);
1321
1322 void ath10k_ce_disable_interrupts(struct ath10k *ar)
1323 {
1324         int ce_id;
1325
1326         for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1327                 ath10k_ce_disable_interrupt(ar, ce_id);
1328 }
1329 EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1330
1331 void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)
1332 {
1333         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1334         struct ath10k_ce_pipe *ce_state;
1335
1336         ce_state  = &ce->ce_states[ce_id];
1337         if (ce_state->attr_flags & CE_ATTR_POLL)
1338                 return;
1339
1340         ath10k_ce_per_engine_handler_adjust(ce_state);
1341 }
1342 EXPORT_SYMBOL(ath10k_ce_enable_interrupt);
1343
1344 void ath10k_ce_enable_interrupts(struct ath10k *ar)
1345 {
1346         int ce_id;
1347
1348         /* Enable interrupts for copy engine that
1349          * are not using polling mode.
1350          */
1351         for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1352                 ath10k_ce_enable_interrupt(ar, ce_id);
1353 }
1354 EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1355
1356 static int ath10k_ce_init_src_ring(struct ath10k *ar,
1357                                    unsigned int ce_id,
1358                                    const struct ce_attr *attr)
1359 {
1360         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1361         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1362         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1363         u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1364
1365         nentries = roundup_pow_of_two(attr->src_nentries);
1366
1367         if (ar->hw_params.target_64bit)
1368                 memset(src_ring->base_addr_owner_space, 0,
1369                        nentries * sizeof(struct ce_desc_64));
1370         else
1371                 memset(src_ring->base_addr_owner_space, 0,
1372                        nentries * sizeof(struct ce_desc));
1373
1374         src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1375         src_ring->sw_index &= src_ring->nentries_mask;
1376         src_ring->hw_index = src_ring->sw_index;
1377
1378         src_ring->write_index =
1379                 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1380         src_ring->write_index &= src_ring->nentries_mask;
1381
1382         ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1383                                          src_ring->base_addr_ce_space);
1384         ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1385         ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1386         ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1387         ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1388         ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1389
1390         ath10k_dbg(ar, ATH10K_DBG_BOOT,
1391                    "boot init ce src ring id %d entries %d base_addr %pK\n",
1392                    ce_id, nentries, src_ring->base_addr_owner_space);
1393
1394         return 0;
1395 }
1396
1397 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1398                                     unsigned int ce_id,
1399                                     const struct ce_attr *attr)
1400 {
1401         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1402         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1403         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1404         u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1405
1406         nentries = roundup_pow_of_two(attr->dest_nentries);
1407
1408         if (ar->hw_params.target_64bit)
1409                 memset(dest_ring->base_addr_owner_space, 0,
1410                        nentries * sizeof(struct ce_desc_64));
1411         else
1412                 memset(dest_ring->base_addr_owner_space, 0,
1413                        nentries * sizeof(struct ce_desc));
1414
1415         dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1416         dest_ring->sw_index &= dest_ring->nentries_mask;
1417         dest_ring->write_index =
1418                 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1419         dest_ring->write_index &= dest_ring->nentries_mask;
1420
1421         ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1422                                           dest_ring->base_addr_ce_space);
1423         ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1424         ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1425         ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1426         ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1427
1428         ath10k_dbg(ar, ATH10K_DBG_BOOT,
1429                    "boot ce dest ring id %d entries %d base_addr %pK\n",
1430                    ce_id, nentries, dest_ring->base_addr_owner_space);
1431
1432         return 0;
1433 }
1434
1435 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1436                                        struct ath10k_ce_ring *src_ring,
1437                                        u32 nentries)
1438 {
1439         src_ring->shadow_base_unaligned = kcalloc(nentries,
1440                                                   sizeof(struct ce_desc_64),
1441                                                   GFP_KERNEL);
1442         if (!src_ring->shadow_base_unaligned)
1443                 return -ENOMEM;
1444
1445         src_ring->shadow_base = (struct ce_desc_64 *)
1446                         PTR_ALIGN(src_ring->shadow_base_unaligned,
1447                                   CE_DESC_RING_ALIGN);
1448         return 0;
1449 }
1450
1451 static struct ath10k_ce_ring *
1452 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1453                          const struct ce_attr *attr)
1454 {
1455         struct ath10k_ce_ring *src_ring;
1456         u32 nentries = attr->src_nentries;
1457         dma_addr_t base_addr;
1458         int ret;
1459
1460         nentries = roundup_pow_of_two(nentries);
1461
1462         src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1463                                        nentries), GFP_KERNEL);
1464         if (src_ring == NULL)
1465                 return ERR_PTR(-ENOMEM);
1466
1467         src_ring->nentries = nentries;
1468         src_ring->nentries_mask = nentries - 1;
1469
1470         /*
1471          * Legacy platforms that do not support cache
1472          * coherent DMA are unsupported
1473          */
1474         src_ring->base_addr_owner_space_unaligned =
1475                 dma_alloc_coherent(ar->dev,
1476                                    (nentries * sizeof(struct ce_desc) +
1477                                     CE_DESC_RING_ALIGN),
1478                                    &base_addr, GFP_KERNEL);
1479         if (!src_ring->base_addr_owner_space_unaligned) {
1480                 kfree(src_ring);
1481                 return ERR_PTR(-ENOMEM);
1482         }
1483
1484         src_ring->base_addr_ce_space_unaligned = base_addr;
1485
1486         src_ring->base_addr_owner_space =
1487                         PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1488                                   CE_DESC_RING_ALIGN);
1489         src_ring->base_addr_ce_space =
1490                         ALIGN(src_ring->base_addr_ce_space_unaligned,
1491                               CE_DESC_RING_ALIGN);
1492
1493         if (ar->hw_params.shadow_reg_support) {
1494                 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1495                 if (ret) {
1496                         dma_free_coherent(ar->dev,
1497                                           (nentries * sizeof(struct ce_desc) +
1498                                            CE_DESC_RING_ALIGN),
1499                                           src_ring->base_addr_owner_space_unaligned,
1500                                           base_addr);
1501                         kfree(src_ring);
1502                         return ERR_PTR(ret);
1503                 }
1504         }
1505
1506         return src_ring;
1507 }
1508
1509 static struct ath10k_ce_ring *
1510 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1511                             const struct ce_attr *attr)
1512 {
1513         struct ath10k_ce_ring *src_ring;
1514         u32 nentries = attr->src_nentries;
1515         dma_addr_t base_addr;
1516         int ret;
1517
1518         nentries = roundup_pow_of_two(nentries);
1519
1520         src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1521                                        nentries), GFP_KERNEL);
1522         if (!src_ring)
1523                 return ERR_PTR(-ENOMEM);
1524
1525         src_ring->nentries = nentries;
1526         src_ring->nentries_mask = nentries - 1;
1527
1528         /* Legacy platforms that do not support cache
1529          * coherent DMA are unsupported
1530          */
1531         src_ring->base_addr_owner_space_unaligned =
1532                 dma_alloc_coherent(ar->dev,
1533                                    (nentries * sizeof(struct ce_desc_64) +
1534                                     CE_DESC_RING_ALIGN),
1535                                    &base_addr, GFP_KERNEL);
1536         if (!src_ring->base_addr_owner_space_unaligned) {
1537                 kfree(src_ring);
1538                 return ERR_PTR(-ENOMEM);
1539         }
1540
1541         src_ring->base_addr_ce_space_unaligned = base_addr;
1542
1543         src_ring->base_addr_owner_space =
1544                         PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1545                                   CE_DESC_RING_ALIGN);
1546         src_ring->base_addr_ce_space =
1547                         ALIGN(src_ring->base_addr_ce_space_unaligned,
1548                               CE_DESC_RING_ALIGN);
1549
1550         if (ar->hw_params.shadow_reg_support) {
1551                 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1552                 if (ret) {
1553                         dma_free_coherent(ar->dev,
1554                                           (nentries * sizeof(struct ce_desc_64) +
1555                                            CE_DESC_RING_ALIGN),
1556                                           src_ring->base_addr_owner_space_unaligned,
1557                                           base_addr);
1558                         kfree(src_ring);
1559                         return ERR_PTR(ret);
1560                 }
1561         }
1562
1563         return src_ring;
1564 }
1565
1566 static struct ath10k_ce_ring *
1567 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1568                           const struct ce_attr *attr)
1569 {
1570         struct ath10k_ce_ring *dest_ring;
1571         u32 nentries;
1572         dma_addr_t base_addr;
1573
1574         nentries = roundup_pow_of_two(attr->dest_nentries);
1575
1576         dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1577                                         nentries), GFP_KERNEL);
1578         if (dest_ring == NULL)
1579                 return ERR_PTR(-ENOMEM);
1580
1581         dest_ring->nentries = nentries;
1582         dest_ring->nentries_mask = nentries - 1;
1583
1584         /*
1585          * Legacy platforms that do not support cache
1586          * coherent DMA are unsupported
1587          */
1588         dest_ring->base_addr_owner_space_unaligned =
1589                 dma_alloc_coherent(ar->dev,
1590                                    (nentries * sizeof(struct ce_desc) +
1591                                     CE_DESC_RING_ALIGN),
1592                                    &base_addr, GFP_KERNEL);
1593         if (!dest_ring->base_addr_owner_space_unaligned) {
1594                 kfree(dest_ring);
1595                 return ERR_PTR(-ENOMEM);
1596         }
1597
1598         dest_ring->base_addr_ce_space_unaligned = base_addr;
1599
1600         dest_ring->base_addr_owner_space =
1601                         PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1602                                   CE_DESC_RING_ALIGN);
1603         dest_ring->base_addr_ce_space =
1604                                 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1605                                       CE_DESC_RING_ALIGN);
1606
1607         return dest_ring;
1608 }
1609
1610 static struct ath10k_ce_ring *
1611 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1612                              const struct ce_attr *attr)
1613 {
1614         struct ath10k_ce_ring *dest_ring;
1615         u32 nentries;
1616         dma_addr_t base_addr;
1617
1618         nentries = roundup_pow_of_two(attr->dest_nentries);
1619
1620         dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1621                                         nentries), GFP_KERNEL);
1622         if (!dest_ring)
1623                 return ERR_PTR(-ENOMEM);
1624
1625         dest_ring->nentries = nentries;
1626         dest_ring->nentries_mask = nentries - 1;
1627
1628         /* Legacy platforms that do not support cache
1629          * coherent DMA are unsupported
1630          */
1631         dest_ring->base_addr_owner_space_unaligned =
1632                 dma_alloc_coherent(ar->dev,
1633                                    (nentries * sizeof(struct ce_desc_64) +
1634                                     CE_DESC_RING_ALIGN),
1635                                    &base_addr, GFP_KERNEL);
1636         if (!dest_ring->base_addr_owner_space_unaligned) {
1637                 kfree(dest_ring);
1638                 return ERR_PTR(-ENOMEM);
1639         }
1640
1641         dest_ring->base_addr_ce_space_unaligned = base_addr;
1642
1643         /* Correctly initialize memory to 0 to prevent garbage
1644          * data crashing system when download firmware
1645          */
1646         dest_ring->base_addr_owner_space =
1647                         PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1648                                   CE_DESC_RING_ALIGN);
1649         dest_ring->base_addr_ce_space =
1650                         ALIGN(dest_ring->base_addr_ce_space_unaligned,
1651                               CE_DESC_RING_ALIGN);
1652
1653         return dest_ring;
1654 }
1655
1656 /*
1657  * Initialize a Copy Engine based on caller-supplied attributes.
1658  * This may be called once to initialize both source and destination
1659  * rings or it may be called twice for separate source and destination
1660  * initialization. It may be that only one side or the other is
1661  * initialized by software/firmware.
1662  */
1663 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1664                         const struct ce_attr *attr)
1665 {
1666         int ret;
1667
1668         if (attr->src_nentries) {
1669                 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1670                 if (ret) {
1671                         ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1672                                    ce_id, ret);
1673                         return ret;
1674                 }
1675         }
1676
1677         if (attr->dest_nentries) {
1678                 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1679                 if (ret) {
1680                         ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1681                                    ce_id, ret);
1682                         return ret;
1683                 }
1684         }
1685
1686         return 0;
1687 }
1688 EXPORT_SYMBOL(ath10k_ce_init_pipe);
1689
1690 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1691 {
1692         u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1693
1694         ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1695         ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1696         ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1697         ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1698 }
1699
1700 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1701 {
1702         u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1703
1704         ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1705         ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1706         ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1707 }
1708
1709 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1710 {
1711         ath10k_ce_deinit_src_ring(ar, ce_id);
1712         ath10k_ce_deinit_dest_ring(ar, ce_id);
1713 }
1714 EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1715
1716 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1717 {
1718         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1719         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1720
1721         if (ce_state->src_ring) {
1722                 if (ar->hw_params.shadow_reg_support)
1723                         kfree(ce_state->src_ring->shadow_base_unaligned);
1724                 dma_free_coherent(ar->dev,
1725                                   (ce_state->src_ring->nentries *
1726                                    sizeof(struct ce_desc) +
1727                                    CE_DESC_RING_ALIGN),
1728                                   ce_state->src_ring->base_addr_owner_space,
1729                                   ce_state->src_ring->base_addr_ce_space);
1730                 kfree(ce_state->src_ring);
1731         }
1732
1733         if (ce_state->dest_ring) {
1734                 dma_free_coherent(ar->dev,
1735                                   (ce_state->dest_ring->nentries *
1736                                    sizeof(struct ce_desc) +
1737                                    CE_DESC_RING_ALIGN),
1738                                   ce_state->dest_ring->base_addr_owner_space,
1739                                   ce_state->dest_ring->base_addr_ce_space);
1740                 kfree(ce_state->dest_ring);
1741         }
1742
1743         ce_state->src_ring = NULL;
1744         ce_state->dest_ring = NULL;
1745 }
1746
1747 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1748 {
1749         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1750         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1751
1752         if (ce_state->src_ring) {
1753                 if (ar->hw_params.shadow_reg_support)
1754                         kfree(ce_state->src_ring->shadow_base_unaligned);
1755                 dma_free_coherent(ar->dev,
1756                                   (ce_state->src_ring->nentries *
1757                                    sizeof(struct ce_desc_64) +
1758                                    CE_DESC_RING_ALIGN),
1759                                   ce_state->src_ring->base_addr_owner_space,
1760                                   ce_state->src_ring->base_addr_ce_space);
1761                 kfree(ce_state->src_ring);
1762         }
1763
1764         if (ce_state->dest_ring) {
1765                 dma_free_coherent(ar->dev,
1766                                   (ce_state->dest_ring->nentries *
1767                                    sizeof(struct ce_desc_64) +
1768                                    CE_DESC_RING_ALIGN),
1769                                   ce_state->dest_ring->base_addr_owner_space,
1770                                   ce_state->dest_ring->base_addr_ce_space);
1771                 kfree(ce_state->dest_ring);
1772         }
1773
1774         ce_state->src_ring = NULL;
1775         ce_state->dest_ring = NULL;
1776 }
1777
1778 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1779 {
1780         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1781         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1782
1783         ce_state->ops->ce_free_pipe(ar, ce_id);
1784 }
1785 EXPORT_SYMBOL(ath10k_ce_free_pipe);
1786
1787 void ath10k_ce_dump_registers(struct ath10k *ar,
1788                               struct ath10k_fw_crash_data *crash_data)
1789 {
1790         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1791         struct ath10k_ce_crash_data ce_data;
1792         u32 addr, id;
1793
1794         lockdep_assert_held(&ar->dump_mutex);
1795
1796         ath10k_err(ar, "Copy Engine register dump:\n");
1797
1798         spin_lock_bh(&ce->ce_lock);
1799         for (id = 0; id < CE_COUNT; id++) {
1800                 addr = ath10k_ce_base_address(ar, id);
1801                 ce_data.base_addr = cpu_to_le32(addr);
1802
1803                 ce_data.src_wr_idx =
1804                         cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1805                 ce_data.src_r_idx =
1806                         cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1807                 ce_data.dst_wr_idx =
1808                         cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1809                 ce_data.dst_r_idx =
1810                         cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1811
1812                 if (crash_data)
1813                         crash_data->ce_crash_data[id] = ce_data;
1814
1815                 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1816                            le32_to_cpu(ce_data.base_addr),
1817                            le32_to_cpu(ce_data.src_wr_idx),
1818                            le32_to_cpu(ce_data.src_r_idx),
1819                            le32_to_cpu(ce_data.dst_wr_idx),
1820                            le32_to_cpu(ce_data.dst_r_idx));
1821         }
1822
1823         spin_unlock_bh(&ce->ce_lock);
1824 }
1825 EXPORT_SYMBOL(ath10k_ce_dump_registers);
1826
1827 static const struct ath10k_ce_ops ce_ops = {
1828         .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1829         .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1830         .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1831         .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1832         .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1833         .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1834         .ce_free_pipe = _ath10k_ce_free_pipe,
1835         .ce_send_nolock = _ath10k_ce_send_nolock,
1836         .ce_set_src_ring_base_addr_hi = NULL,
1837         .ce_set_dest_ring_base_addr_hi = NULL,
1838         .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1839 };
1840
1841 static const struct ath10k_ce_ops ce_64_ops = {
1842         .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1843         .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1844         .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1845         .ce_completed_recv_next_nolock =
1846                                 _ath10k_ce_completed_recv_next_nolock_64,
1847         .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1848         .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1849         .ce_free_pipe = _ath10k_ce_free_pipe_64,
1850         .ce_send_nolock = _ath10k_ce_send_nolock_64,
1851         .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1852         .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1853         .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1854 };
1855
1856 static void ath10k_ce_set_ops(struct ath10k *ar,
1857                               struct ath10k_ce_pipe *ce_state)
1858 {
1859         switch (ar->hw_rev) {
1860         case ATH10K_HW_WCN3990:
1861                 ce_state->ops = &ce_64_ops;
1862                 break;
1863         default:
1864                 ce_state->ops = &ce_ops;
1865                 break;
1866         }
1867 }
1868
1869 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1870                          const struct ce_attr *attr)
1871 {
1872         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1873         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1874         int ret;
1875
1876         ath10k_ce_set_ops(ar, ce_state);
1877         /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1878          * additional TX locking checks.
1879          *
1880          * For the lack of a better place do the check here.
1881          */
1882         BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1883                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1884         BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1885                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1886         BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1887                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1888
1889         ce_state->ar = ar;
1890         ce_state->id = ce_id;
1891         ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1892         ce_state->attr_flags = attr->flags;
1893         ce_state->src_sz_max = attr->src_sz_max;
1894
1895         if (attr->src_nentries)
1896                 ce_state->send_cb = attr->send_cb;
1897
1898         if (attr->dest_nentries)
1899                 ce_state->recv_cb = attr->recv_cb;
1900
1901         if (attr->src_nentries) {
1902                 ce_state->src_ring =
1903                         ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1904                 if (IS_ERR(ce_state->src_ring)) {
1905                         ret = PTR_ERR(ce_state->src_ring);
1906                         ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1907                                    ce_id, ret);
1908                         ce_state->src_ring = NULL;
1909                         return ret;
1910                 }
1911         }
1912
1913         if (attr->dest_nentries) {
1914                 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1915                                                                         ce_id,
1916                                                                         attr);
1917                 if (IS_ERR(ce_state->dest_ring)) {
1918                         ret = PTR_ERR(ce_state->dest_ring);
1919                         ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1920                                    ce_id, ret);
1921                         ce_state->dest_ring = NULL;
1922                         return ret;
1923                 }
1924         }
1925
1926         return 0;
1927 }
1928 EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1929
1930 void ath10k_ce_alloc_rri(struct ath10k *ar)
1931 {
1932         int i;
1933         u32 value;
1934         u32 ctrl1_regs;
1935         u32 ce_base_addr;
1936         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1937
1938         ce->vaddr_rri = dma_alloc_coherent(ar->dev,
1939                                            (CE_COUNT * sizeof(u32)),
1940                                            &ce->paddr_rri, GFP_KERNEL);
1941
1942         if (!ce->vaddr_rri)
1943                 return;
1944
1945         ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
1946                           lower_32_bits(ce->paddr_rri));
1947         ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
1948                           (upper_32_bits(ce->paddr_rri) &
1949                           CE_DESC_ADDR_HI_MASK));
1950
1951         for (i = 0; i < CE_COUNT; i++) {
1952                 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
1953                 ce_base_addr = ath10k_ce_base_address(ar, i);
1954                 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
1955                 value |= ar->hw_ce_regs->upd->mask;
1956                 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
1957         }
1958 }
1959 EXPORT_SYMBOL(ath10k_ce_alloc_rri);
1960
1961 void ath10k_ce_free_rri(struct ath10k *ar)
1962 {
1963         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1964
1965         dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
1966                           ce->vaddr_rri,
1967                           ce->paddr_rri);
1968 }
1969 EXPORT_SYMBOL(ath10k_ce_free_rri);