1 // SPDX-License-Identifier: ISC
3 * Copyright (c) 2005-2011 Atheros Communications Inc.
4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
5 * Copyright (c) 2018 The Linux Foundation. All rights reserved.
6 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
14 * Support for Copy Engine hardware, which is mainly used for
15 * communication between Host and Target over a PCIe interconnect.
19 * A single CopyEngine (CE) comprises two "rings":
23 * Each ring consists of a number of descriptors which specify
24 * an address, length, and meta-data.
26 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
27 * controls one ring and the other side controls the other ring.
28 * The source side chooses when to initiate a transfer and it
29 * chooses what to send (buffer address, length). The destination
30 * side keeps a supply of "anonymous receive buffers" available and
31 * it handles incoming data as it arrives (when the destination
32 * receives an interrupt).
34 * The sender may send a simple buffer (address/length) or it may
35 * send a small list of buffers. When a small list is sent, hardware
36 * "gathers" these and they end up in a single destination buffer
37 * with a single interrupt.
39 * There are several "contexts" managed by this layer -- more, it
40 * may seem -- than should be needed. These are provided mainly for
41 * maximum flexibility and especially to facilitate a simpler HIF
42 * implementation. There are per-CopyEngine recv, send, and watermark
43 * contexts. These are supplied by the caller when a recv, send,
44 * or watermark handler is established and they are echoed back to
45 * the caller when the respective callbacks are invoked. There is
46 * also a per-transfer context supplied by the caller when a buffer
47 * (or sendlist) is sent and when a buffer is enqueued for recv.
48 * These per-transfer contexts are echoed back to the caller when
49 * the buffer is sent/received.
52 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
53 struct ath10k_ce_pipe *ce_state)
55 u32 ce_id = ce_state->id;
75 ath10k_warn(ar, "invalid CE id: %d", ce_id);
81 static inline unsigned int
82 ath10k_set_ring_byte(unsigned int offset,
83 struct ath10k_hw_ce_regs_addr_map *addr_map)
85 return ((offset << addr_map->lsb) & addr_map->mask);
88 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
90 struct ath10k_ce *ce = ath10k_ce_priv(ar);
92 return ce->bus_ops->read32(ar, offset);
95 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
97 struct ath10k_ce *ce = ath10k_ce_priv(ar);
99 ce->bus_ops->write32(ar, offset, value);
102 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
106 ath10k_ce_write32(ar, ce_ctrl_addr +
107 ar->hw_ce_regs->dst_wr_index_addr, n);
110 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
113 return ath10k_ce_read32(ar, ce_ctrl_addr +
114 ar->hw_ce_regs->dst_wr_index_addr);
117 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
121 ath10k_ce_write32(ar, ce_ctrl_addr +
122 ar->hw_ce_regs->sr_wr_index_addr, n);
125 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
128 return ath10k_ce_read32(ar, ce_ctrl_addr +
129 ar->hw_ce_regs->sr_wr_index_addr);
132 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
135 struct ath10k_ce *ce = ath10k_ce_priv(ar);
137 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
140 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
143 struct ath10k_ce *ce = ath10k_ce_priv(ar);
144 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
145 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
148 if (ar->hw_params.rri_on_ddr &&
149 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
150 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
152 index = ath10k_ce_read32(ar, ce_ctrl_addr +
153 ar->hw_ce_regs->current_srri_addr);
159 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
160 struct ath10k_ce_pipe *ce_state,
163 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
166 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
170 struct ath10k_ce *ce = ath10k_ce_priv(ar);
171 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
172 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
173 u32 addr_lo = lower_32_bits(addr);
175 ath10k_ce_write32(ar, ce_ctrl_addr +
176 ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
178 if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
179 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
184 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
188 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
190 ath10k_ce_write32(ar, ce_ctrl_addr +
191 ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
194 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
198 ath10k_ce_write32(ar, ce_ctrl_addr +
199 ar->hw_ce_regs->sr_size_addr, n);
202 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
206 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
208 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
211 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
212 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) |
213 ath10k_set_ring_byte(n, ctrl_regs->dmax));
216 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
220 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
222 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
225 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
226 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
227 ath10k_set_ring_byte(n, ctrl_regs->src_ring));
230 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
234 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
236 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
239 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
240 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
241 ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
245 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
247 struct ath10k_ce *ce = ath10k_ce_priv(ar);
249 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
253 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
256 struct ath10k_ce *ce = ath10k_ce_priv(ar);
257 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
258 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
261 if (ar->hw_params.rri_on_ddr &&
262 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
263 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
265 index = ath10k_ce_read32(ar, ce_ctrl_addr +
266 ar->hw_ce_regs->current_drri_addr);
271 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
275 struct ath10k_ce *ce = ath10k_ce_priv(ar);
276 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
277 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
278 u32 addr_lo = lower_32_bits(addr);
280 ath10k_ce_write32(ar, ce_ctrl_addr +
281 ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
283 if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
284 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
289 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
293 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
296 reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
297 ar->hw_ce_regs->dr_base_addr_hi);
298 reg_value &= ~CE_DESC_ADDR_HI_MASK;
299 reg_value |= addr_hi;
300 ath10k_ce_write32(ar, ce_ctrl_addr +
301 ar->hw_ce_regs->dr_base_addr_hi, reg_value);
304 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
308 ath10k_ce_write32(ar, ce_ctrl_addr +
309 ar->hw_ce_regs->dr_size_addr, n);
312 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
316 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
317 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
319 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
320 (addr & ~(srcr_wm->wm_high->mask)) |
321 (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
324 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
328 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
329 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
331 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
332 (addr & ~(srcr_wm->wm_low->mask)) |
333 (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
336 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
340 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
341 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
343 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
344 (addr & ~(dstr_wm->wm_high->mask)) |
345 (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
348 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
352 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
353 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
355 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
356 (addr & ~(dstr_wm->wm_low->mask)) |
357 (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
360 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
363 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
365 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
366 ar->hw_ce_regs->host_ie_addr);
368 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
369 host_ie_addr | host_ie->copy_complete->mask);
372 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
375 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
377 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
378 ar->hw_ce_regs->host_ie_addr);
380 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
381 host_ie_addr & ~(host_ie->copy_complete->mask));
384 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
387 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
389 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
390 ar->hw_ce_regs->host_ie_addr);
392 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
393 host_ie_addr & ~(wm_regs->wm_mask));
396 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
399 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
401 u32 misc_ie_addr = ath10k_ce_read32(ar,
402 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
404 ath10k_ce_write32(ar,
405 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
406 misc_ie_addr & ~(misc_regs->err_mask));
409 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
413 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
415 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
419 * Guts of ath10k_ce_send.
420 * The caller takes responsibility for any needed locking.
422 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
423 void *per_transfer_context,
426 unsigned int transfer_id,
429 struct ath10k *ar = ce_state->ar;
430 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
431 struct ce_desc *desc, sdesc;
432 unsigned int nentries_mask = src_ring->nentries_mask;
433 unsigned int sw_index = src_ring->sw_index;
434 unsigned int write_index = src_ring->write_index;
435 u32 ctrl_addr = ce_state->ctrl_addr;
439 if (nbytes > ce_state->src_sz_max)
440 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
441 __func__, nbytes, ce_state->src_sz_max);
443 if (unlikely(CE_RING_DELTA(nentries_mask,
444 write_index, sw_index - 1) <= 0)) {
449 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
452 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
454 if (flags & CE_SEND_FLAG_GATHER)
455 desc_flags |= CE_DESC_FLAGS_GATHER;
456 if (flags & CE_SEND_FLAG_BYTE_SWAP)
457 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
459 sdesc.addr = __cpu_to_le32(buffer);
460 sdesc.nbytes = __cpu_to_le16(nbytes);
461 sdesc.flags = __cpu_to_le16(desc_flags);
465 src_ring->per_transfer_context[write_index] = per_transfer_context;
467 /* Update Source Ring Write Index */
468 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
471 if (!(flags & CE_SEND_FLAG_GATHER))
472 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
474 src_ring->write_index = write_index;
479 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
480 void *per_transfer_context,
483 unsigned int transfer_id,
486 struct ath10k *ar = ce_state->ar;
487 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
488 struct ce_desc_64 *desc, sdesc;
489 unsigned int nentries_mask = src_ring->nentries_mask;
490 unsigned int sw_index;
491 unsigned int write_index = src_ring->write_index;
492 u32 ctrl_addr = ce_state->ctrl_addr;
497 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
500 if (nbytes > ce_state->src_sz_max)
501 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
502 __func__, nbytes, ce_state->src_sz_max);
504 if (ar->hw_params.rri_on_ddr)
505 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
507 sw_index = src_ring->sw_index;
509 if (unlikely(CE_RING_DELTA(nentries_mask,
510 write_index, sw_index - 1) <= 0)) {
515 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
518 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
520 if (flags & CE_SEND_FLAG_GATHER)
521 desc_flags |= CE_DESC_FLAGS_GATHER;
523 if (flags & CE_SEND_FLAG_BYTE_SWAP)
524 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
526 addr = (__le32 *)&sdesc.addr;
528 flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
529 addr[0] = __cpu_to_le32(buffer);
530 addr[1] = __cpu_to_le32(flags);
531 if (flags & CE_SEND_FLAG_GATHER)
532 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
534 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
536 sdesc.nbytes = __cpu_to_le16(nbytes);
537 sdesc.flags = __cpu_to_le16(desc_flags);
541 src_ring->per_transfer_context[write_index] = per_transfer_context;
543 /* Update Source Ring Write Index */
544 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
546 if (!(flags & CE_SEND_FLAG_GATHER)) {
547 if (ar->hw_params.shadow_reg_support)
548 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
551 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
555 src_ring->write_index = write_index;
560 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
561 void *per_transfer_context,
564 unsigned int transfer_id,
567 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
568 buffer, nbytes, transfer_id, flags);
570 EXPORT_SYMBOL(ath10k_ce_send_nolock);
572 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
574 struct ath10k *ar = pipe->ar;
575 struct ath10k_ce *ce = ath10k_ce_priv(ar);
576 struct ath10k_ce_ring *src_ring = pipe->src_ring;
577 u32 ctrl_addr = pipe->ctrl_addr;
579 lockdep_assert_held(&ce->ce_lock);
582 * This function must be called only if there is an incomplete
583 * scatter-gather transfer (before index register is updated)
584 * that needs to be cleaned up.
586 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
589 if (WARN_ON_ONCE(src_ring->write_index ==
590 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
593 src_ring->write_index--;
594 src_ring->write_index &= src_ring->nentries_mask;
596 src_ring->per_transfer_context[src_ring->write_index] = NULL;
598 EXPORT_SYMBOL(__ath10k_ce_send_revert);
600 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
601 void *per_transfer_context,
604 unsigned int transfer_id,
607 struct ath10k *ar = ce_state->ar;
608 struct ath10k_ce *ce = ath10k_ce_priv(ar);
611 spin_lock_bh(&ce->ce_lock);
612 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
613 buffer, nbytes, transfer_id, flags);
614 spin_unlock_bh(&ce->ce_lock);
618 EXPORT_SYMBOL(ath10k_ce_send);
620 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
622 struct ath10k *ar = pipe->ar;
623 struct ath10k_ce *ce = ath10k_ce_priv(ar);
626 spin_lock_bh(&ce->ce_lock);
627 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
628 pipe->src_ring->write_index,
629 pipe->src_ring->sw_index - 1);
630 spin_unlock_bh(&ce->ce_lock);
634 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
636 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
638 struct ath10k *ar = pipe->ar;
639 struct ath10k_ce *ce = ath10k_ce_priv(ar);
640 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
641 unsigned int nentries_mask = dest_ring->nentries_mask;
642 unsigned int write_index = dest_ring->write_index;
643 unsigned int sw_index = dest_ring->sw_index;
645 lockdep_assert_held(&ce->ce_lock);
647 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
649 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
651 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
654 struct ath10k *ar = pipe->ar;
655 struct ath10k_ce *ce = ath10k_ce_priv(ar);
656 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
657 unsigned int nentries_mask = dest_ring->nentries_mask;
658 unsigned int write_index = dest_ring->write_index;
659 unsigned int sw_index = dest_ring->sw_index;
660 struct ce_desc *base = dest_ring->base_addr_owner_space;
661 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
662 u32 ctrl_addr = pipe->ctrl_addr;
664 lockdep_assert_held(&ce->ce_lock);
666 if ((pipe->id != 5) &&
667 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
670 desc->addr = __cpu_to_le32(paddr);
673 dest_ring->per_transfer_context[write_index] = ctx;
674 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
675 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
676 dest_ring->write_index = write_index;
681 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
685 struct ath10k *ar = pipe->ar;
686 struct ath10k_ce *ce = ath10k_ce_priv(ar);
687 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
688 unsigned int nentries_mask = dest_ring->nentries_mask;
689 unsigned int write_index = dest_ring->write_index;
690 unsigned int sw_index = dest_ring->sw_index;
691 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
692 struct ce_desc_64 *desc =
693 CE_DEST_RING_TO_DESC_64(base, write_index);
694 u32 ctrl_addr = pipe->ctrl_addr;
696 lockdep_assert_held(&ce->ce_lock);
698 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
701 desc->addr = __cpu_to_le64(paddr);
702 desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
706 dest_ring->per_transfer_context[write_index] = ctx;
707 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
708 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
709 dest_ring->write_index = write_index;
714 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
716 struct ath10k *ar = pipe->ar;
717 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
718 unsigned int nentries_mask = dest_ring->nentries_mask;
719 unsigned int write_index = dest_ring->write_index;
720 u32 ctrl_addr = pipe->ctrl_addr;
721 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
723 /* Prevent CE ring stuck issue that will occur when ring is full.
724 * Make sure that write index is 1 less than read index.
726 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
729 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
730 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
731 dest_ring->write_index = write_index;
733 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
735 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
738 struct ath10k *ar = pipe->ar;
739 struct ath10k_ce *ce = ath10k_ce_priv(ar);
742 spin_lock_bh(&ce->ce_lock);
743 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
744 spin_unlock_bh(&ce->ce_lock);
748 EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
751 * Guts of ath10k_ce_completed_recv_next.
752 * The caller takes responsibility for any necessary locking.
755 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
756 void **per_transfer_contextp,
757 unsigned int *nbytesp)
759 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
760 unsigned int nentries_mask = dest_ring->nentries_mask;
761 unsigned int sw_index = dest_ring->sw_index;
763 struct ce_desc *base = dest_ring->base_addr_owner_space;
764 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
765 struct ce_desc sdesc;
768 /* Copy in one go for performance reasons */
771 nbytes = __le16_to_cpu(sdesc.nbytes);
774 * This closes a relatively unusual race where the Host
775 * sees the updated DRRI before the update to the
776 * corresponding descriptor has completed. We treat this
777 * as a descriptor that is not yet done.
784 /* Return data from completed destination descriptor */
787 if (per_transfer_contextp)
788 *per_transfer_contextp =
789 dest_ring->per_transfer_context[sw_index];
791 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
792 * So update transfer context all CEs except CE5.
794 if (ce_state->id != 5)
795 dest_ring->per_transfer_context[sw_index] = NULL;
797 /* Update sw_index */
798 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
799 dest_ring->sw_index = sw_index;
805 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
806 void **per_transfer_contextp,
807 unsigned int *nbytesp)
809 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
810 unsigned int nentries_mask = dest_ring->nentries_mask;
811 unsigned int sw_index = dest_ring->sw_index;
812 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
813 struct ce_desc_64 *desc =
814 CE_DEST_RING_TO_DESC_64(base, sw_index);
815 struct ce_desc_64 sdesc;
818 /* Copy in one go for performance reasons */
821 nbytes = __le16_to_cpu(sdesc.nbytes);
823 /* This closes a relatively unusual race where the Host
824 * sees the updated DRRI before the update to the
825 * corresponding descriptor has completed. We treat this
826 * as a descriptor that is not yet done.
833 /* Return data from completed destination descriptor */
836 if (per_transfer_contextp)
837 *per_transfer_contextp =
838 dest_ring->per_transfer_context[sw_index];
840 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
841 * So update transfer context all CEs except CE5.
843 if (ce_state->id != 5)
844 dest_ring->per_transfer_context[sw_index] = NULL;
846 /* Update sw_index */
847 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
848 dest_ring->sw_index = sw_index;
853 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
854 void **per_transfer_ctx,
855 unsigned int *nbytesp)
857 return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
861 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
863 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
864 void **per_transfer_contextp,
865 unsigned int *nbytesp)
867 struct ath10k *ar = ce_state->ar;
868 struct ath10k_ce *ce = ath10k_ce_priv(ar);
871 spin_lock_bh(&ce->ce_lock);
872 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
873 per_transfer_contextp,
876 spin_unlock_bh(&ce->ce_lock);
880 EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
882 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
883 void **per_transfer_contextp,
886 struct ath10k_ce_ring *dest_ring;
887 unsigned int nentries_mask;
888 unsigned int sw_index;
889 unsigned int write_index;
892 struct ath10k_ce *ce;
894 dest_ring = ce_state->dest_ring;
900 ce = ath10k_ce_priv(ar);
902 spin_lock_bh(&ce->ce_lock);
904 nentries_mask = dest_ring->nentries_mask;
905 sw_index = dest_ring->sw_index;
906 write_index = dest_ring->write_index;
907 if (write_index != sw_index) {
908 struct ce_desc *base = dest_ring->base_addr_owner_space;
909 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
911 /* Return data from completed destination descriptor */
912 *bufferp = __le32_to_cpu(desc->addr);
914 if (per_transfer_contextp)
915 *per_transfer_contextp =
916 dest_ring->per_transfer_context[sw_index];
919 dest_ring->per_transfer_context[sw_index] = NULL;
922 /* Update sw_index */
923 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
924 dest_ring->sw_index = sw_index;
930 spin_unlock_bh(&ce->ce_lock);
935 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
936 void **per_transfer_contextp,
939 struct ath10k_ce_ring *dest_ring;
940 unsigned int nentries_mask;
941 unsigned int sw_index;
942 unsigned int write_index;
945 struct ath10k_ce *ce;
947 dest_ring = ce_state->dest_ring;
953 ce = ath10k_ce_priv(ar);
955 spin_lock_bh(&ce->ce_lock);
957 nentries_mask = dest_ring->nentries_mask;
958 sw_index = dest_ring->sw_index;
959 write_index = dest_ring->write_index;
960 if (write_index != sw_index) {
961 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
962 struct ce_desc_64 *desc =
963 CE_DEST_RING_TO_DESC_64(base, sw_index);
965 /* Return data from completed destination descriptor */
966 *bufferp = __le64_to_cpu(desc->addr);
968 if (per_transfer_contextp)
969 *per_transfer_contextp =
970 dest_ring->per_transfer_context[sw_index];
973 dest_ring->per_transfer_context[sw_index] = NULL;
976 /* Update sw_index */
977 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
978 dest_ring->sw_index = sw_index;
984 spin_unlock_bh(&ce->ce_lock);
989 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
990 void **per_transfer_contextp,
993 return ce_state->ops->ce_revoke_recv_next(ce_state,
994 per_transfer_contextp,
997 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
1000 * Guts of ath10k_ce_completed_send_next.
1001 * The caller takes responsibility for any necessary locking.
1003 static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1004 void **per_transfer_contextp)
1006 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1007 u32 ctrl_addr = ce_state->ctrl_addr;
1008 struct ath10k *ar = ce_state->ar;
1009 unsigned int nentries_mask = src_ring->nentries_mask;
1010 unsigned int sw_index = src_ring->sw_index;
1011 unsigned int read_index;
1012 struct ce_desc *desc;
1014 if (src_ring->hw_index == sw_index) {
1016 * The SW completion index has caught up with the cached
1017 * version of the HW completion index.
1018 * Update the cached HW completion index to see whether
1019 * the SW has really caught up to the HW, or if the cached
1020 * value of the HW index has become stale.
1023 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1024 if (read_index == 0xffffffff)
1027 read_index &= nentries_mask;
1028 src_ring->hw_index = read_index;
1031 if (ar->hw_params.rri_on_ddr)
1032 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1034 read_index = src_ring->hw_index;
1036 if (read_index == sw_index)
1039 if (per_transfer_contextp)
1040 *per_transfer_contextp =
1041 src_ring->per_transfer_context[sw_index];
1044 src_ring->per_transfer_context[sw_index] = NULL;
1045 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1049 /* Update sw_index */
1050 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1051 src_ring->sw_index = sw_index;
1056 static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1057 void **per_transfer_contextp)
1059 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1060 u32 ctrl_addr = ce_state->ctrl_addr;
1061 struct ath10k *ar = ce_state->ar;
1062 unsigned int nentries_mask = src_ring->nentries_mask;
1063 unsigned int sw_index = src_ring->sw_index;
1064 unsigned int read_index;
1065 struct ce_desc_64 *desc;
1067 if (src_ring->hw_index == sw_index) {
1069 * The SW completion index has caught up with the cached
1070 * version of the HW completion index.
1071 * Update the cached HW completion index to see whether
1072 * the SW has really caught up to the HW, or if the cached
1073 * value of the HW index has become stale.
1076 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1077 if (read_index == 0xffffffff)
1080 read_index &= nentries_mask;
1081 src_ring->hw_index = read_index;
1084 if (ar->hw_params.rri_on_ddr)
1085 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1087 read_index = src_ring->hw_index;
1089 if (read_index == sw_index)
1092 if (per_transfer_contextp)
1093 *per_transfer_contextp =
1094 src_ring->per_transfer_context[sw_index];
1097 src_ring->per_transfer_context[sw_index] = NULL;
1098 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1102 /* Update sw_index */
1103 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1104 src_ring->sw_index = sw_index;
1109 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1110 void **per_transfer_contextp)
1112 return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1113 per_transfer_contextp);
1115 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1117 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1118 struct ath10k_ce_ring *src_ring,
1120 dma_addr_t *bufferp,
1124 struct ce_desc *base = src_ring->base_addr_owner_space;
1125 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1127 /* Return data from completed source descriptor */
1128 *bufferp = __le32_to_cpu(desc->addr);
1129 *nbytesp = __le16_to_cpu(desc->nbytes);
1130 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1131 CE_DESC_FLAGS_META_DATA);
1134 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1135 struct ath10k_ce_ring *src_ring,
1137 dma_addr_t *bufferp,
1141 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1142 struct ce_desc_64 *desc =
1143 CE_SRC_RING_TO_DESC_64(base, sw_index);
1145 /* Return data from completed source descriptor */
1146 *bufferp = __le64_to_cpu(desc->addr);
1147 *nbytesp = __le16_to_cpu(desc->nbytes);
1148 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1149 CE_DESC_FLAGS_META_DATA);
1152 /* NB: Modeled after ath10k_ce_completed_send_next */
1153 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1154 void **per_transfer_contextp,
1155 dma_addr_t *bufferp,
1156 unsigned int *nbytesp,
1157 unsigned int *transfer_idp)
1159 struct ath10k_ce_ring *src_ring;
1160 unsigned int nentries_mask;
1161 unsigned int sw_index;
1162 unsigned int write_index;
1165 struct ath10k_ce *ce;
1167 src_ring = ce_state->src_ring;
1173 ce = ath10k_ce_priv(ar);
1175 spin_lock_bh(&ce->ce_lock);
1177 nentries_mask = src_ring->nentries_mask;
1178 sw_index = src_ring->sw_index;
1179 write_index = src_ring->write_index;
1181 if (write_index != sw_index) {
1182 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1186 if (per_transfer_contextp)
1187 *per_transfer_contextp =
1188 src_ring->per_transfer_context[sw_index];
1191 src_ring->per_transfer_context[sw_index] = NULL;
1193 /* Update sw_index */
1194 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1195 src_ring->sw_index = sw_index;
1201 spin_unlock_bh(&ce->ce_lock);
1205 EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1207 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1208 void **per_transfer_contextp)
1210 struct ath10k *ar = ce_state->ar;
1211 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1214 spin_lock_bh(&ce->ce_lock);
1215 ret = ath10k_ce_completed_send_next_nolock(ce_state,
1216 per_transfer_contextp);
1217 spin_unlock_bh(&ce->ce_lock);
1221 EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1224 * Guts of interrupt handler for per-engine interrupts on a particular CE.
1226 * Invokes registered callbacks for recv_complete,
1227 * send_complete, and watermarks.
1229 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1231 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1232 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1233 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1234 u32 ctrl_addr = ce_state->ctrl_addr;
1237 * Clear before handling
1239 * Misc CE interrupts are not being handled, but still need
1242 * NOTE: When the last copy engine interrupt is cleared the
1243 * hardware will go to sleep. Once this happens any access to
1244 * the CE registers can cause a hardware fault.
1246 ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1247 wm_regs->cc_mask | wm_regs->wm_mask);
1249 if (ce_state->recv_cb)
1250 ce_state->recv_cb(ce_state);
1252 if (ce_state->send_cb)
1253 ce_state->send_cb(ce_state);
1255 EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1258 * Handler for per-engine interrupts on ALL active CEs.
1259 * This is used in cases where the system is sharing a
1260 * single interrupt for all CEs
1263 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1268 intr_summary = ath10k_ce_interrupt_summary(ar);
1270 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1271 if (intr_summary & (1 << ce_id))
1272 intr_summary &= ~(1 << ce_id);
1274 /* no intr pending on this CE */
1277 ath10k_ce_per_engine_service(ar, ce_id);
1280 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1283 * Adjust interrupts for the copy complete handler.
1284 * If it's needed for either send or recv, then unmask
1285 * this interrupt; otherwise, mask it.
1287 * Called with ce_lock held.
1289 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1291 u32 ctrl_addr = ce_state->ctrl_addr;
1292 struct ath10k *ar = ce_state->ar;
1293 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1295 if ((!disable_copy_compl_intr) &&
1296 (ce_state->send_cb || ce_state->recv_cb))
1297 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1299 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1301 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1304 void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)
1306 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1307 struct ath10k_ce_pipe *ce_state;
1310 ce_state = &ce->ce_states[ce_id];
1311 if (ce_state->attr_flags & CE_ATTR_POLL)
1314 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1316 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1317 ath10k_ce_error_intr_disable(ar, ctrl_addr);
1318 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1320 EXPORT_SYMBOL(ath10k_ce_disable_interrupt);
1322 void ath10k_ce_disable_interrupts(struct ath10k *ar)
1326 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1327 ath10k_ce_disable_interrupt(ar, ce_id);
1329 EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1331 void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)
1333 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1334 struct ath10k_ce_pipe *ce_state;
1336 ce_state = &ce->ce_states[ce_id];
1337 if (ce_state->attr_flags & CE_ATTR_POLL)
1340 ath10k_ce_per_engine_handler_adjust(ce_state);
1342 EXPORT_SYMBOL(ath10k_ce_enable_interrupt);
1344 void ath10k_ce_enable_interrupts(struct ath10k *ar)
1348 /* Enable interrupts for copy engine that
1349 * are not using polling mode.
1351 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1352 ath10k_ce_enable_interrupt(ar, ce_id);
1354 EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1356 static int ath10k_ce_init_src_ring(struct ath10k *ar,
1358 const struct ce_attr *attr)
1360 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1361 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1362 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1363 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1365 nentries = roundup_pow_of_two(attr->src_nentries);
1367 if (ar->hw_params.target_64bit)
1368 memset(src_ring->base_addr_owner_space, 0,
1369 nentries * sizeof(struct ce_desc_64));
1371 memset(src_ring->base_addr_owner_space, 0,
1372 nentries * sizeof(struct ce_desc));
1374 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1375 src_ring->sw_index &= src_ring->nentries_mask;
1376 src_ring->hw_index = src_ring->sw_index;
1378 src_ring->write_index =
1379 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1380 src_ring->write_index &= src_ring->nentries_mask;
1382 ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1383 src_ring->base_addr_ce_space);
1384 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1385 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1386 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1387 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1388 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1390 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1391 "boot init ce src ring id %d entries %d base_addr %pK\n",
1392 ce_id, nentries, src_ring->base_addr_owner_space);
1397 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1399 const struct ce_attr *attr)
1401 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1402 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1403 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1404 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1406 nentries = roundup_pow_of_two(attr->dest_nentries);
1408 if (ar->hw_params.target_64bit)
1409 memset(dest_ring->base_addr_owner_space, 0,
1410 nentries * sizeof(struct ce_desc_64));
1412 memset(dest_ring->base_addr_owner_space, 0,
1413 nentries * sizeof(struct ce_desc));
1415 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1416 dest_ring->sw_index &= dest_ring->nentries_mask;
1417 dest_ring->write_index =
1418 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1419 dest_ring->write_index &= dest_ring->nentries_mask;
1421 ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1422 dest_ring->base_addr_ce_space);
1423 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1424 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1425 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1426 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1428 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1429 "boot ce dest ring id %d entries %d base_addr %pK\n",
1430 ce_id, nentries, dest_ring->base_addr_owner_space);
1435 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1436 struct ath10k_ce_ring *src_ring,
1439 src_ring->shadow_base_unaligned = kcalloc(nentries,
1440 sizeof(struct ce_desc_64),
1442 if (!src_ring->shadow_base_unaligned)
1445 src_ring->shadow_base = (struct ce_desc_64 *)
1446 PTR_ALIGN(src_ring->shadow_base_unaligned,
1447 CE_DESC_RING_ALIGN);
1451 static struct ath10k_ce_ring *
1452 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1453 const struct ce_attr *attr)
1455 struct ath10k_ce_ring *src_ring;
1456 u32 nentries = attr->src_nentries;
1457 dma_addr_t base_addr;
1460 nentries = roundup_pow_of_two(nentries);
1462 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1463 nentries), GFP_KERNEL);
1464 if (src_ring == NULL)
1465 return ERR_PTR(-ENOMEM);
1467 src_ring->nentries = nentries;
1468 src_ring->nentries_mask = nentries - 1;
1471 * Legacy platforms that do not support cache
1472 * coherent DMA are unsupported
1474 src_ring->base_addr_owner_space_unaligned =
1475 dma_alloc_coherent(ar->dev,
1476 (nentries * sizeof(struct ce_desc) +
1477 CE_DESC_RING_ALIGN),
1478 &base_addr, GFP_KERNEL);
1479 if (!src_ring->base_addr_owner_space_unaligned) {
1481 return ERR_PTR(-ENOMEM);
1484 src_ring->base_addr_ce_space_unaligned = base_addr;
1486 src_ring->base_addr_owner_space =
1487 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1488 CE_DESC_RING_ALIGN);
1489 src_ring->base_addr_ce_space =
1490 ALIGN(src_ring->base_addr_ce_space_unaligned,
1491 CE_DESC_RING_ALIGN);
1493 if (ar->hw_params.shadow_reg_support) {
1494 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1496 dma_free_coherent(ar->dev,
1497 (nentries * sizeof(struct ce_desc) +
1498 CE_DESC_RING_ALIGN),
1499 src_ring->base_addr_owner_space_unaligned,
1502 return ERR_PTR(ret);
1509 static struct ath10k_ce_ring *
1510 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1511 const struct ce_attr *attr)
1513 struct ath10k_ce_ring *src_ring;
1514 u32 nentries = attr->src_nentries;
1515 dma_addr_t base_addr;
1518 nentries = roundup_pow_of_two(nentries);
1520 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1521 nentries), GFP_KERNEL);
1523 return ERR_PTR(-ENOMEM);
1525 src_ring->nentries = nentries;
1526 src_ring->nentries_mask = nentries - 1;
1528 /* Legacy platforms that do not support cache
1529 * coherent DMA are unsupported
1531 src_ring->base_addr_owner_space_unaligned =
1532 dma_alloc_coherent(ar->dev,
1533 (nentries * sizeof(struct ce_desc_64) +
1534 CE_DESC_RING_ALIGN),
1535 &base_addr, GFP_KERNEL);
1536 if (!src_ring->base_addr_owner_space_unaligned) {
1538 return ERR_PTR(-ENOMEM);
1541 src_ring->base_addr_ce_space_unaligned = base_addr;
1543 src_ring->base_addr_owner_space =
1544 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1545 CE_DESC_RING_ALIGN);
1546 src_ring->base_addr_ce_space =
1547 ALIGN(src_ring->base_addr_ce_space_unaligned,
1548 CE_DESC_RING_ALIGN);
1550 if (ar->hw_params.shadow_reg_support) {
1551 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1553 dma_free_coherent(ar->dev,
1554 (nentries * sizeof(struct ce_desc_64) +
1555 CE_DESC_RING_ALIGN),
1556 src_ring->base_addr_owner_space_unaligned,
1559 return ERR_PTR(ret);
1566 static struct ath10k_ce_ring *
1567 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1568 const struct ce_attr *attr)
1570 struct ath10k_ce_ring *dest_ring;
1572 dma_addr_t base_addr;
1574 nentries = roundup_pow_of_two(attr->dest_nentries);
1576 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1577 nentries), GFP_KERNEL);
1578 if (dest_ring == NULL)
1579 return ERR_PTR(-ENOMEM);
1581 dest_ring->nentries = nentries;
1582 dest_ring->nentries_mask = nentries - 1;
1585 * Legacy platforms that do not support cache
1586 * coherent DMA are unsupported
1588 dest_ring->base_addr_owner_space_unaligned =
1589 dma_alloc_coherent(ar->dev,
1590 (nentries * sizeof(struct ce_desc) +
1591 CE_DESC_RING_ALIGN),
1592 &base_addr, GFP_KERNEL);
1593 if (!dest_ring->base_addr_owner_space_unaligned) {
1595 return ERR_PTR(-ENOMEM);
1598 dest_ring->base_addr_ce_space_unaligned = base_addr;
1600 dest_ring->base_addr_owner_space =
1601 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1602 CE_DESC_RING_ALIGN);
1603 dest_ring->base_addr_ce_space =
1604 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1605 CE_DESC_RING_ALIGN);
1610 static struct ath10k_ce_ring *
1611 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1612 const struct ce_attr *attr)
1614 struct ath10k_ce_ring *dest_ring;
1616 dma_addr_t base_addr;
1618 nentries = roundup_pow_of_two(attr->dest_nentries);
1620 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1621 nentries), GFP_KERNEL);
1623 return ERR_PTR(-ENOMEM);
1625 dest_ring->nentries = nentries;
1626 dest_ring->nentries_mask = nentries - 1;
1628 /* Legacy platforms that do not support cache
1629 * coherent DMA are unsupported
1631 dest_ring->base_addr_owner_space_unaligned =
1632 dma_alloc_coherent(ar->dev,
1633 (nentries * sizeof(struct ce_desc_64) +
1634 CE_DESC_RING_ALIGN),
1635 &base_addr, GFP_KERNEL);
1636 if (!dest_ring->base_addr_owner_space_unaligned) {
1638 return ERR_PTR(-ENOMEM);
1641 dest_ring->base_addr_ce_space_unaligned = base_addr;
1643 /* Correctly initialize memory to 0 to prevent garbage
1644 * data crashing system when download firmware
1646 dest_ring->base_addr_owner_space =
1647 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1648 CE_DESC_RING_ALIGN);
1649 dest_ring->base_addr_ce_space =
1650 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1651 CE_DESC_RING_ALIGN);
1657 * Initialize a Copy Engine based on caller-supplied attributes.
1658 * This may be called once to initialize both source and destination
1659 * rings or it may be called twice for separate source and destination
1660 * initialization. It may be that only one side or the other is
1661 * initialized by software/firmware.
1663 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1664 const struct ce_attr *attr)
1668 if (attr->src_nentries) {
1669 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1671 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1677 if (attr->dest_nentries) {
1678 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1680 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1688 EXPORT_SYMBOL(ath10k_ce_init_pipe);
1690 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1692 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1694 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1695 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1696 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1697 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1700 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1702 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1704 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1705 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1706 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1709 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1711 ath10k_ce_deinit_src_ring(ar, ce_id);
1712 ath10k_ce_deinit_dest_ring(ar, ce_id);
1714 EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1716 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1718 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1719 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1721 if (ce_state->src_ring) {
1722 if (ar->hw_params.shadow_reg_support)
1723 kfree(ce_state->src_ring->shadow_base_unaligned);
1724 dma_free_coherent(ar->dev,
1725 (ce_state->src_ring->nentries *
1726 sizeof(struct ce_desc) +
1727 CE_DESC_RING_ALIGN),
1728 ce_state->src_ring->base_addr_owner_space,
1729 ce_state->src_ring->base_addr_ce_space);
1730 kfree(ce_state->src_ring);
1733 if (ce_state->dest_ring) {
1734 dma_free_coherent(ar->dev,
1735 (ce_state->dest_ring->nentries *
1736 sizeof(struct ce_desc) +
1737 CE_DESC_RING_ALIGN),
1738 ce_state->dest_ring->base_addr_owner_space,
1739 ce_state->dest_ring->base_addr_ce_space);
1740 kfree(ce_state->dest_ring);
1743 ce_state->src_ring = NULL;
1744 ce_state->dest_ring = NULL;
1747 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1749 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1750 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1752 if (ce_state->src_ring) {
1753 if (ar->hw_params.shadow_reg_support)
1754 kfree(ce_state->src_ring->shadow_base_unaligned);
1755 dma_free_coherent(ar->dev,
1756 (ce_state->src_ring->nentries *
1757 sizeof(struct ce_desc_64) +
1758 CE_DESC_RING_ALIGN),
1759 ce_state->src_ring->base_addr_owner_space,
1760 ce_state->src_ring->base_addr_ce_space);
1761 kfree(ce_state->src_ring);
1764 if (ce_state->dest_ring) {
1765 dma_free_coherent(ar->dev,
1766 (ce_state->dest_ring->nentries *
1767 sizeof(struct ce_desc_64) +
1768 CE_DESC_RING_ALIGN),
1769 ce_state->dest_ring->base_addr_owner_space,
1770 ce_state->dest_ring->base_addr_ce_space);
1771 kfree(ce_state->dest_ring);
1774 ce_state->src_ring = NULL;
1775 ce_state->dest_ring = NULL;
1778 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1780 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1781 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1783 ce_state->ops->ce_free_pipe(ar, ce_id);
1785 EXPORT_SYMBOL(ath10k_ce_free_pipe);
1787 void ath10k_ce_dump_registers(struct ath10k *ar,
1788 struct ath10k_fw_crash_data *crash_data)
1790 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1791 struct ath10k_ce_crash_data ce_data;
1794 lockdep_assert_held(&ar->dump_mutex);
1796 ath10k_err(ar, "Copy Engine register dump:\n");
1798 spin_lock_bh(&ce->ce_lock);
1799 for (id = 0; id < CE_COUNT; id++) {
1800 addr = ath10k_ce_base_address(ar, id);
1801 ce_data.base_addr = cpu_to_le32(addr);
1803 ce_data.src_wr_idx =
1804 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1806 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1807 ce_data.dst_wr_idx =
1808 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1810 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1813 crash_data->ce_crash_data[id] = ce_data;
1815 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1816 le32_to_cpu(ce_data.base_addr),
1817 le32_to_cpu(ce_data.src_wr_idx),
1818 le32_to_cpu(ce_data.src_r_idx),
1819 le32_to_cpu(ce_data.dst_wr_idx),
1820 le32_to_cpu(ce_data.dst_r_idx));
1823 spin_unlock_bh(&ce->ce_lock);
1825 EXPORT_SYMBOL(ath10k_ce_dump_registers);
1827 static const struct ath10k_ce_ops ce_ops = {
1828 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1829 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1830 .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1831 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1832 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1833 .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1834 .ce_free_pipe = _ath10k_ce_free_pipe,
1835 .ce_send_nolock = _ath10k_ce_send_nolock,
1836 .ce_set_src_ring_base_addr_hi = NULL,
1837 .ce_set_dest_ring_base_addr_hi = NULL,
1838 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1841 static const struct ath10k_ce_ops ce_64_ops = {
1842 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1843 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1844 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1845 .ce_completed_recv_next_nolock =
1846 _ath10k_ce_completed_recv_next_nolock_64,
1847 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1848 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1849 .ce_free_pipe = _ath10k_ce_free_pipe_64,
1850 .ce_send_nolock = _ath10k_ce_send_nolock_64,
1851 .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1852 .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1853 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1856 static void ath10k_ce_set_ops(struct ath10k *ar,
1857 struct ath10k_ce_pipe *ce_state)
1859 switch (ar->hw_rev) {
1860 case ATH10K_HW_WCN3990:
1861 ce_state->ops = &ce_64_ops;
1864 ce_state->ops = &ce_ops;
1869 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1870 const struct ce_attr *attr)
1872 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1873 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1876 ath10k_ce_set_ops(ar, ce_state);
1877 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1878 * additional TX locking checks.
1880 * For the lack of a better place do the check here.
1882 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1883 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1884 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1885 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1886 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1887 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1890 ce_state->id = ce_id;
1891 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1892 ce_state->attr_flags = attr->flags;
1893 ce_state->src_sz_max = attr->src_sz_max;
1895 if (attr->src_nentries)
1896 ce_state->send_cb = attr->send_cb;
1898 if (attr->dest_nentries)
1899 ce_state->recv_cb = attr->recv_cb;
1901 if (attr->src_nentries) {
1902 ce_state->src_ring =
1903 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1904 if (IS_ERR(ce_state->src_ring)) {
1905 ret = PTR_ERR(ce_state->src_ring);
1906 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1908 ce_state->src_ring = NULL;
1913 if (attr->dest_nentries) {
1914 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1917 if (IS_ERR(ce_state->dest_ring)) {
1918 ret = PTR_ERR(ce_state->dest_ring);
1919 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1921 ce_state->dest_ring = NULL;
1928 EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1930 void ath10k_ce_alloc_rri(struct ath10k *ar)
1936 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1938 ce->vaddr_rri = dma_alloc_coherent(ar->dev,
1939 (CE_COUNT * sizeof(u32)),
1940 &ce->paddr_rri, GFP_KERNEL);
1945 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
1946 lower_32_bits(ce->paddr_rri));
1947 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
1948 (upper_32_bits(ce->paddr_rri) &
1949 CE_DESC_ADDR_HI_MASK));
1951 for (i = 0; i < CE_COUNT; i++) {
1952 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
1953 ce_base_addr = ath10k_ce_base_address(ar, i);
1954 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
1955 value |= ar->hw_ce_regs->upd->mask;
1956 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
1959 EXPORT_SYMBOL(ath10k_ce_alloc_rri);
1961 void ath10k_ce_free_rri(struct ath10k *ar)
1963 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1965 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
1969 EXPORT_SYMBOL(ath10k_ce_free_rri);