2 * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of Freescale Semiconductor nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
16 * ALTERNATIVELY, this software may be distributed under the terms of the
17 * GNU General Public License ("GPL") as published by the Free Software
18 * Foundation, either version 2 of that License or (at your option) any
21 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
22 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
24 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
25 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
26 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
28 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
30 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 #include <asm/cacheflush.h>
35 #include <linux/slab.h>
36 #include "../../include/dpaa2-global.h"
38 #include "qbman-portal.h"
40 #define QMAN_REV_4000 0x04000000
41 #define QMAN_REV_4100 0x04010000
42 #define QMAN_REV_4101 0x04010001
43 #define QMAN_REV_MASK 0xffff0000
45 /* All QBMan command and result structures use this "valid bit" encoding */
46 #define QB_VALID_BIT ((u32)0x80)
48 /* QBMan portal management command codes */
49 #define QBMAN_MC_ACQUIRE 0x30
50 #define QBMAN_WQCHAN_CONFIGURE 0x46
52 /* CINH register offsets */
53 #define QBMAN_CINH_SWP_EQAR 0x8c0
54 #define QBMAN_CINH_SWP_DQPI 0xa00
55 #define QBMAN_CINH_SWP_DCAP 0xac0
56 #define QBMAN_CINH_SWP_SDQCR 0xb00
57 #define QBMAN_CINH_SWP_RAR 0xcc0
58 #define QBMAN_CINH_SWP_ISR 0xe00
59 #define QBMAN_CINH_SWP_IER 0xe40
60 #define QBMAN_CINH_SWP_ISDR 0xe80
61 #define QBMAN_CINH_SWP_IIR 0xec0
63 /* CENA register offsets */
64 #define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((u32)(n) << 6))
65 #define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((u32)(n) << 6))
66 #define QBMAN_CENA_SWP_RCR(n) (0x400 + ((u32)(n) << 6))
67 #define QBMAN_CENA_SWP_CR 0x600
68 #define QBMAN_CENA_SWP_RR(vb) (0x700 + ((u32)(vb) >> 1))
69 #define QBMAN_CENA_SWP_VDQCR 0x780
71 /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */
72 #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)(p) & 0x1ff) >> 6)
74 /* Define token used to determine if response written to memory is valid */
75 #define QMAN_DQ_TOKEN_VALID 1
77 /* SDQCR attribute codes */
78 #define QB_SDQCR_FC_SHIFT 29
79 #define QB_SDQCR_FC_MASK 0x1
80 #define QB_SDQCR_DCT_SHIFT 24
81 #define QB_SDQCR_DCT_MASK 0x3
82 #define QB_SDQCR_TOK_SHIFT 16
83 #define QB_SDQCR_TOK_MASK 0xff
84 #define QB_SDQCR_SRC_SHIFT 0
85 #define QB_SDQCR_SRC_MASK 0xffff
87 /* opaque token for static dequeues */
88 #define QMAN_SDQCR_TOKEN 0xbb
90 enum qbman_sdqcr_dct {
91 qbman_sdqcr_dct_null = 0,
92 qbman_sdqcr_dct_prio_ics,
93 qbman_sdqcr_dct_active_ics,
94 qbman_sdqcr_dct_active
98 qbman_sdqcr_fc_one = 0,
99 qbman_sdqcr_fc_up_to_3 = 1
104 static inline u32 qbman_read_register(struct qbman_swp *p, u32 offset)
106 return readl_relaxed(p->addr_cinh + offset);
109 static inline void qbman_write_register(struct qbman_swp *p, u32 offset,
112 writel_relaxed(value, p->addr_cinh + offset);
115 static inline void *qbman_get_cmd(struct qbman_swp *p, u32 offset)
117 return p->addr_cena + offset;
120 #define QBMAN_CINH_SWP_CFG 0xd00
122 #define SWP_CFG_DQRR_MF_SHIFT 20
123 #define SWP_CFG_EST_SHIFT 16
124 #define SWP_CFG_WN_SHIFT 14
125 #define SWP_CFG_RPM_SHIFT 12
126 #define SWP_CFG_DCM_SHIFT 10
127 #define SWP_CFG_EPM_SHIFT 8
128 #define SWP_CFG_SD_SHIFT 5
129 #define SWP_CFG_SP_SHIFT 4
130 #define SWP_CFG_SE_SHIFT 3
131 #define SWP_CFG_DP_SHIFT 2
132 #define SWP_CFG_DE_SHIFT 1
133 #define SWP_CFG_EP_SHIFT 0
135 static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn, u8 est, u8 rpm, u8 dcm,
136 u8 epm, int sd, int sp, int se,
137 int dp, int de, int ep)
139 return (max_fill << SWP_CFG_DQRR_MF_SHIFT |
140 est << SWP_CFG_EST_SHIFT |
141 wn << SWP_CFG_WN_SHIFT |
142 rpm << SWP_CFG_RPM_SHIFT |
143 dcm << SWP_CFG_DCM_SHIFT |
144 epm << SWP_CFG_EPM_SHIFT |
145 sd << SWP_CFG_SD_SHIFT |
146 sp << SWP_CFG_SP_SHIFT |
147 se << SWP_CFG_SE_SHIFT |
148 dp << SWP_CFG_DP_SHIFT |
149 de << SWP_CFG_DE_SHIFT |
150 ep << SWP_CFG_EP_SHIFT);
154 * qbman_swp_init() - Create a functional object representing the given
155 * QBMan portal descriptor.
156 * @d: the given qbman swp descriptor
158 * Return qbman_swp portal for success, NULL if the object cannot
161 struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
163 struct qbman_swp *p = kmalloc(sizeof(*p), GFP_KERNEL);
169 p->mc.valid_bit = QB_VALID_BIT;
171 p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
172 p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
173 p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
175 atomic_set(&p->vdq.available, 1);
176 p->vdq.valid_bit = QB_VALID_BIT;
177 p->dqrr.next_idx = 0;
178 p->dqrr.valid_bit = QB_VALID_BIT;
180 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_4100) {
181 p->dqrr.dqrr_size = 4;
182 p->dqrr.reset_bug = 1;
184 p->dqrr.dqrr_size = 8;
185 p->dqrr.reset_bug = 0;
188 p->addr_cena = d->cena_bar;
189 p->addr_cinh = d->cinh_bar;
191 reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
192 1, /* Writes Non-cacheable */
193 0, /* EQCR_CI stashing threshold */
194 3, /* RPM: Valid bit mode, RCR in array mode */
195 2, /* DCM: Discrete consumption ack mode */
196 3, /* EPM: Valid bit mode, EQCR in array mode */
197 0, /* mem stashing drop enable == FALSE */
198 1, /* mem stashing priority == TRUE */
199 0, /* mem stashing enable == FALSE */
200 1, /* dequeue stashing priority == TRUE */
201 0, /* dequeue stashing enable == FALSE */
202 0); /* EQCR_CI stashing priority == FALSE */
204 qbman_write_register(p, QBMAN_CINH_SWP_CFG, reg);
205 reg = qbman_read_register(p, QBMAN_CINH_SWP_CFG);
207 pr_err("qbman: the portal is not enabled!\n");
212 * SDQCR needs to be initialized to 0 when no channels are
213 * being dequeued from or else the QMan HW will indicate an
214 * error. The values that were calculated above will be
215 * applied when dequeues from a specific channel are enabled.
217 qbman_write_register(p, QBMAN_CINH_SWP_SDQCR, 0);
222 * qbman_swp_finish() - Create and destroy a functional object representing
223 * the given QBMan portal descriptor.
224 * @p: the qbman_swp object to be destroyed
226 void qbman_swp_finish(struct qbman_swp *p)
232 * qbman_swp_interrupt_read_status()
233 * @p: the given software portal
235 * Return the value in the SWP_ISR register.
237 u32 qbman_swp_interrupt_read_status(struct qbman_swp *p)
239 return qbman_read_register(p, QBMAN_CINH_SWP_ISR);
243 * qbman_swp_interrupt_clear_status()
244 * @p: the given software portal
245 * @mask: The mask to clear in SWP_ISR register
247 void qbman_swp_interrupt_clear_status(struct qbman_swp *p, u32 mask)
249 qbman_write_register(p, QBMAN_CINH_SWP_ISR, mask);
253 * qbman_swp_interrupt_get_trigger() - read interrupt enable register
254 * @p: the given software portal
256 * Return the value in the SWP_IER register.
258 u32 qbman_swp_interrupt_get_trigger(struct qbman_swp *p)
260 return qbman_read_register(p, QBMAN_CINH_SWP_IER);
264 * qbman_swp_interrupt_set_trigger() - enable interrupts for a swp
265 * @p: the given software portal
266 * @mask: The mask of bits to enable in SWP_IER
268 void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, u32 mask)
270 qbman_write_register(p, QBMAN_CINH_SWP_IER, mask);
274 * qbman_swp_interrupt_get_inhibit() - read interrupt mask register
275 * @p: the given software portal object
277 * Return the value in the SWP_IIR register.
279 int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p)
281 return qbman_read_register(p, QBMAN_CINH_SWP_IIR);
285 * qbman_swp_interrupt_set_inhibit() - write interrupt mask register
286 * @p: the given software portal object
287 * @mask: The mask to set in SWP_IIR register
289 void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit)
291 qbman_write_register(p, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0);
295 * Different management commands all use this common base layer of code to issue
296 * commands and poll for results.
300 * Returns a pointer to where the caller should fill in their management command
301 * (caller should ignore the verb byte)
303 void *qbman_swp_mc_start(struct qbman_swp *p)
305 return qbman_get_cmd(p, QBMAN_CENA_SWP_CR);
309 * Commits merges in the caller-supplied command verb (which should not include
310 * the valid-bit) and submits the command to hardware
312 void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, u8 cmd_verb)
317 *v = cmd_verb | p->mc.valid_bit;
321 * Checks for a completed response (returns non-NULL if only if the response
324 void *qbman_swp_mc_result(struct qbman_swp *p)
328 ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR(p->mc.valid_bit));
330 /* Remove the valid-bit - command completed if the rest is non-zero */
331 verb = ret[0] & ~QB_VALID_BIT;
334 p->mc.valid_bit ^= QB_VALID_BIT;
338 #define QB_ENQUEUE_CMD_OPTIONS_SHIFT 0
339 enum qb_enqueue_commands {
341 enqueue_response_always = 1,
342 enqueue_rejects_to_fq = 2
345 #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT 2
346 #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3
347 #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT 4
350 * qbman_eq_desc_clear() - Clear the contents of a descriptor to
351 * default/starting state.
353 void qbman_eq_desc_clear(struct qbman_eq_desc *d)
355 memset(d, 0, sizeof(*d));
359 * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp
360 * @d: the enqueue descriptor.
361 * @response_success: 1 = enqueue with response always; 0 = enqueue with
362 * rejections returned on a FQ.
364 void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success)
366 d->verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT);
368 d->verb |= enqueue_response_always;
370 d->verb |= enqueue_rejects_to_fq;
374 * Exactly one of the following descriptor "targets" should be set. (Calling any
375 * one of these will replace the effect of any prior call to one of these.)
376 * -enqueue to a frame queue
377 * -enqueue to a queuing destination
381 * qbman_eq_desc_set_fq() - set the FQ for the enqueue command
382 * @d: the enqueue descriptor
383 * @fqid: the id of the frame queue to be enqueued
385 void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, u32 fqid)
387 d->verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT);
388 d->tgtid = cpu_to_le32(fqid);
392 * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command
393 * @d: the enqueue descriptor
394 * @qdid: the id of the queuing destination to be enqueued
395 * @qd_bin: the queuing destination bin
396 * @qd_prio: the queuing destination priority
398 void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, u32 qdid,
399 u32 qd_bin, u32 qd_prio)
401 d->verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT;
402 d->tgtid = cpu_to_le32(qdid);
403 d->qdbin = cpu_to_le16(qd_bin);
407 #define EQAR_IDX(eqar) ((eqar) & 0x7)
408 #define EQAR_VB(eqar) ((eqar) & 0x80)
409 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
412 * qbman_swp_enqueue() - Issue an enqueue command
413 * @s: the software portal used for enqueue
414 * @d: the enqueue descriptor
415 * @fd: the frame descriptor to be enqueued
417 * Please note that 'fd' should only be NULL if the "action" of the
418 * descriptor is "orp_hole" or "orp_nesn".
420 * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
422 int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d,
423 const struct dpaa2_fd *fd)
425 struct qbman_eq_desc *p;
426 u32 eqar = qbman_read_register(s, QBMAN_CINH_SWP_EQAR);
428 if (!EQAR_SUCCESS(eqar))
431 p = qbman_get_cmd(s, QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
432 memcpy(&p->dca, &d->dca, 31);
433 memcpy(&p->fd, fd, sizeof(*fd));
435 /* Set the verb byte, have to substitute in the valid-bit */
437 p->verb = d->verb | EQAR_VB(eqar);
442 /* Static (push) dequeue */
445 * qbman_swp_push_get() - Get the push dequeue setup
446 * @p: the software portal object
447 * @channel_idx: the channel index to query
448 * @enabled: returned boolean to show whether the push dequeue is enabled
449 * for the given channel
451 void qbman_swp_push_get(struct qbman_swp *s, u8 channel_idx, int *enabled)
453 u16 src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
455 WARN_ON(channel_idx > 15);
456 *enabled = src | (1 << channel_idx);
460 * qbman_swp_push_set() - Enable or disable push dequeue
461 * @p: the software portal object
462 * @channel_idx: the channel index (0 to 15)
463 * @enable: enable or disable push dequeue
465 void qbman_swp_push_set(struct qbman_swp *s, u8 channel_idx, int enable)
469 WARN_ON(channel_idx > 15);
471 s->sdq |= 1 << channel_idx;
473 s->sdq &= ~(1 << channel_idx);
475 /* Read make the complete src map. If no channels are enabled
476 * the SDQCR must be 0 or else QMan will assert errors
478 dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
480 qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, s->sdq);
482 qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, 0);
485 #define QB_VDQCR_VERB_DCT_SHIFT 0
486 #define QB_VDQCR_VERB_DT_SHIFT 2
487 #define QB_VDQCR_VERB_RLS_SHIFT 4
488 #define QB_VDQCR_VERB_WAE_SHIFT 5
492 qb_pull_dt_workqueue,
493 qb_pull_dt_framequeue
497 * qbman_pull_desc_clear() - Clear the contents of a descriptor to
498 * default/starting state
499 * @d: the pull dequeue descriptor to be cleared
501 void qbman_pull_desc_clear(struct qbman_pull_desc *d)
503 memset(d, 0, sizeof(*d));
507 * qbman_pull_desc_set_storage()- Set the pull dequeue storage
508 * @d: the pull dequeue descriptor to be set
509 * @storage: the pointer of the memory to store the dequeue result
510 * @storage_phys: the physical address of the storage memory
511 * @stash: to indicate whether write allocate is enabled
513 * If not called, or if called with 'storage' as NULL, the result pull dequeues
514 * will produce results to DQRR. If 'storage' is non-NULL, then results are
515 * produced to the given memory location (using the DMA address which
516 * the caller provides in 'storage_phys'), and 'stash' controls whether or not
517 * those writes to main-memory express a cache-warming attribute.
519 void qbman_pull_desc_set_storage(struct qbman_pull_desc *d,
520 struct dpaa2_dq *storage,
521 dma_addr_t storage_phys,
524 /* save the virtual address */
525 d->rsp_addr_virt = (u64)storage;
528 d->verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT);
531 d->verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT;
533 d->verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT;
535 d->verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
537 d->rsp_addr = cpu_to_le64(storage_phys);
541 * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued
542 * @d: the pull dequeue descriptor to be set
543 * @numframes: number of frames to be set, must be between 1 and 16, inclusive
545 void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, u8 numframes)
547 d->numf = numframes - 1;
550 void qbman_pull_desc_set_token(struct qbman_pull_desc *d, u8 token)
556 * Exactly one of the following descriptor "actions" should be set. (Calling any
557 * one of these will replace the effect of any prior call to one of these.)
558 * - pull dequeue from the given frame queue (FQ)
559 * - pull dequeue from any FQ in the given work queue (WQ)
560 * - pull dequeue from any FQ in any WQ in the given channel
564 * qbman_pull_desc_set_fq() - Set fqid from which the dequeue command dequeues
565 * @fqid: the frame queue index of the given FQ
567 void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, u32 fqid)
569 d->verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT;
570 d->verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT;
571 d->dq_src = cpu_to_le32(fqid);
575 * qbman_pull_desc_set_wq() - Set wqid from which the dequeue command dequeues
576 * @wqid: composed of channel id and wqid within the channel
577 * @dct: the dequeue command type
579 void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, u32 wqid,
580 enum qbman_pull_type_e dct)
582 d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
583 d->verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT;
584 d->dq_src = cpu_to_le32(wqid);
588 * qbman_pull_desc_set_channel() - Set channelid from which the dequeue command
590 * @chid: the channel id to be dequeued
591 * @dct: the dequeue command type
593 void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid,
594 enum qbman_pull_type_e dct)
596 d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
597 d->verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT;
598 d->dq_src = cpu_to_le32(chid);
602 * qbman_swp_pull() - Issue the pull dequeue command
603 * @s: the software portal object
604 * @d: the software portal descriptor which has been configured with
605 * the set of qbman_pull_desc_set_*() calls
607 * Return 0 for success, and -EBUSY if the software portal is not ready
608 * to do pull dequeue.
610 int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
612 struct qbman_pull_desc *p;
614 if (!atomic_dec_and_test(&s->vdq.available)) {
615 atomic_inc(&s->vdq.available);
618 s->vdq.storage = (void *)d->rsp_addr_virt;
619 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
621 p->tok = QMAN_DQ_TOKEN_VALID;
622 p->dq_src = d->dq_src;
623 p->rsp_addr = d->rsp_addr;
624 p->rsp_addr_virt = d->rsp_addr_virt;
627 /* Set the verb byte, have to substitute in the valid-bit */
628 p->verb = d->verb | s->vdq.valid_bit;
629 s->vdq.valid_bit ^= QB_VALID_BIT;
634 #define QMAN_DQRR_PI_MASK 0xf
637 * qbman_swp_dqrr_next() - Get an valid DQRR entry
638 * @s: the software portal object
640 * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
641 * only once, so repeated calls can return a sequence of DQRR entries, without
642 * requiring they be consumed immediately or in any particular order.
644 const struct dpaa2_dq *qbman_swp_dqrr_next(struct qbman_swp *s)
651 /* Before using valid-bit to detect if something is there, we have to
652 * handle the case of the DQRR reset bug...
654 if (unlikely(s->dqrr.reset_bug)) {
656 * We pick up new entries by cache-inhibited producer index,
657 * which means that a non-coherent mapping would require us to
658 * invalidate and read *only* once that PI has indicated that
659 * there's an entry here. The first trip around the DQRR ring
660 * will be much less efficient than all subsequent trips around
663 u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
666 /* there are new entries if pi != next_idx */
667 if (pi == s->dqrr.next_idx)
671 * if next_idx is/was the last ring index, and 'pi' is
672 * different, we can disable the workaround as all the ring
673 * entries have now been DMA'd to so valid-bit checking is
674 * repaired. Note: this logic needs to be based on next_idx
675 * (which increments one at a time), rather than on pi (which
676 * can burst and wrap-around between our snapshots of it).
678 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
679 pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
680 s->dqrr.next_idx, pi);
681 s->dqrr.reset_bug = 0;
683 prefetch(qbman_get_cmd(s,
684 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
687 p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
691 * If the valid-bit isn't of the expected polarity, nothing there. Note,
692 * in the DQRR reset bug workaround, we shouldn't need to skip these
693 * check, because we've already determined that a new entry is available
694 * and we've invalidated the cacheline before reading it, so the
695 * valid-bit behaviour is repaired and should tell us what we already
696 * knew from reading PI.
698 if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
699 prefetch(qbman_get_cmd(s,
700 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
704 * There's something there. Move "next_idx" attention to the next ring
705 * entry (and prefetch it) before returning what we found.
708 s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
709 if (!s->dqrr.next_idx)
710 s->dqrr.valid_bit ^= QB_VALID_BIT;
713 * If this is the final response to a volatile dequeue command
714 * indicate that the vdq is available
717 response_verb = verb & QBMAN_RESULT_MASK;
718 if ((response_verb == QBMAN_RESULT_DQ) &&
719 (flags & DPAA2_DQ_STAT_VOLATILE) &&
720 (flags & DPAA2_DQ_STAT_EXPIRED))
721 atomic_inc(&s->vdq.available);
723 prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
729 * qbman_swp_dqrr_consume() - Consume DQRR entries previously returned from
730 * qbman_swp_dqrr_next().
731 * @s: the software portal object
732 * @dq: the DQRR entry to be consumed
734 void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct dpaa2_dq *dq)
736 qbman_write_register(s, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
740 * qbman_result_has_new_result() - Check and get the dequeue response from the
741 * dq storage memory set in pull dequeue command
742 * @s: the software portal object
743 * @dq: the dequeue result read from the memory
745 * Return 1 for getting a valid dequeue result, or 0 for not getting a valid
748 * Only used for user-provided storage of dequeue results, not DQRR. For
749 * efficiency purposes, the driver will perform any required endianness
750 * conversion to ensure that the user's dequeue result storage is in host-endian
751 * format. As such, once the user has called qbman_result_has_new_result() and
752 * been returned a valid dequeue result, they should not call it again on
753 * the same memory location (except of course if another dequeue command has
754 * been executed to produce a new result to that location).
756 int qbman_result_has_new_result(struct qbman_swp *s, const struct dpaa2_dq *dq)
758 if (dq->dq.tok != QMAN_DQ_TOKEN_VALID)
762 * Set token to be 0 so we will detect change back to 1
763 * next time the looping is traversed. Const is cast away here
764 * as we want users to treat the dequeue responses as read only.
766 ((struct dpaa2_dq *)dq)->dq.tok = 0;
769 * Determine whether VDQCR is available based on whether the
770 * current result is sitting in the first storage location of
773 if (s->vdq.storage == dq) {
774 s->vdq.storage = NULL;
775 atomic_inc(&s->vdq.available);
782 * qbman_release_desc_clear() - Clear the contents of a descriptor to
783 * default/starting state.
785 void qbman_release_desc_clear(struct qbman_release_desc *d)
787 memset(d, 0, sizeof(*d));
788 d->verb = 1 << 5; /* Release Command Valid */
792 * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to
794 void qbman_release_desc_set_bpid(struct qbman_release_desc *d, u16 bpid)
796 d->bpid = cpu_to_le16(bpid);
800 * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI
801 * interrupt source should be asserted after the release command is completed.
803 void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
808 d->verb &= ~(1 << 6);
811 #define RAR_IDX(rar) ((rar) & 0x7)
812 #define RAR_VB(rar) ((rar) & 0x80)
813 #define RAR_SUCCESS(rar) ((rar) & 0x100)
816 * qbman_swp_release() - Issue a buffer release command
817 * @s: the software portal object
818 * @d: the release descriptor
819 * @buffers: a pointer pointing to the buffer address to be released
820 * @num_buffers: number of buffers to be released, must be less than 8
822 * Return 0 for success, -EBUSY if the release command ring is not ready.
824 int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d,
825 const u64 *buffers, unsigned int num_buffers)
828 struct qbman_release_desc *p;
831 if (!num_buffers || (num_buffers > 7))
834 rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
835 if (!RAR_SUCCESS(rar))
838 /* Start the release command */
839 p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
840 /* Copy the caller's buffer pointers to the command */
841 for (i = 0; i < num_buffers; i++)
842 p->buf[i] = cpu_to_le64(buffers[i]);
846 * Set the verb byte, have to substitute in the valid-bit and the number
850 p->verb = d->verb | RAR_VB(rar) | num_buffers;
855 struct qbman_acquire_desc {
863 struct qbman_acquire_rslt {
873 * qbman_swp_acquire() - Issue a buffer acquire command
874 * @s: the software portal object
875 * @bpid: the buffer pool index
876 * @buffers: a pointer pointing to the acquired buffer addresses
877 * @num_buffers: number of buffers to be acquired, must be less than 8
879 * Return 0 for success, or negative error code if the acquire command
882 int qbman_swp_acquire(struct qbman_swp *s, u16 bpid, u64 *buffers,
883 unsigned int num_buffers)
885 struct qbman_acquire_desc *p;
886 struct qbman_acquire_rslt *r;
889 if (!num_buffers || (num_buffers > 7))
892 /* Start the management command */
893 p = qbman_swp_mc_start(s);
898 /* Encode the caller-provided attributes */
899 p->bpid = cpu_to_le16(bpid);
900 p->num = num_buffers;
902 /* Complete the management command */
903 r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE);
905 pr_err("qbman: acquire from BPID %d failed, no response\n",
910 /* Decode the outcome */
911 WARN_ON((r->verb & 0x7f) != QBMAN_MC_ACQUIRE);
913 /* Determine success or failure */
914 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
915 pr_err("qbman: acquire from BPID 0x%x failed, code=0x%02x\n",
920 WARN_ON(r->num > num_buffers);
922 /* Copy the acquired buffers to the caller's array */
923 for (i = 0; i < r->num; i++)
924 buffers[i] = le64_to_cpu(r->buf[i]);
929 struct qbman_alt_fq_state_desc {
936 struct qbman_alt_fq_state_rslt {
942 #define ALT_FQ_FQID_MASK 0x00FFFFFF
944 int qbman_swp_alt_fq_state(struct qbman_swp *s, u32 fqid,
947 struct qbman_alt_fq_state_desc *p;
948 struct qbman_alt_fq_state_rslt *r;
950 /* Start the management command */
951 p = qbman_swp_mc_start(s);
955 p->fqid = cpu_to_le32(fqid) & ALT_FQ_FQID_MASK;
957 /* Complete the management command */
958 r = qbman_swp_mc_complete(s, p, alt_fq_verb);
960 pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n",
965 /* Decode the outcome */
966 WARN_ON((r->verb & QBMAN_RESULT_MASK) != alt_fq_verb);
968 /* Determine success or failure */
969 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
970 pr_err("qbman: ALT FQID %d failed: verb = 0x%08x code = 0x%02x\n",
971 fqid, r->verb, r->rslt);
978 struct qbman_cdan_ctrl_desc {
990 struct qbman_cdan_ctrl_rslt {
997 int qbman_swp_CDAN_set(struct qbman_swp *s, u16 channelid,
998 u8 we_mask, u8 cdan_en,
1001 struct qbman_cdan_ctrl_desc *p = NULL;
1002 struct qbman_cdan_ctrl_rslt *r = NULL;
1004 /* Start the management command */
1005 p = qbman_swp_mc_start(s);
1009 /* Encode the caller-provided attributes */
1010 p->ch = cpu_to_le16(channelid);
1016 p->cdan_ctx = cpu_to_le64(ctx);
1018 /* Complete the management command */
1019 r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE);
1021 pr_err("qbman: wqchan config failed, no response\n");
1025 WARN_ON((r->verb & 0x7f) != QBMAN_WQCHAN_CONFIGURE);
1027 /* Determine success or failure */
1028 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1029 pr_err("qbman: CDAN cQID %d failed: code = 0x%02x\n",
1030 channelid, r->rslt);