-/*-
- * BSD LICENSE
+/* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
+ * Copyright 2018 NXP
*
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Freescale Semiconductor nor the
- * names of its contributors may be used to endorse or promote products
- * derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+#include "qbman_sys.h"
#include "qbman_portal.h"
/* QBMan portal management command codes */
#define QBMAN_MC_ACQUIRE 0x30
#define QBMAN_WQCHAN_CONFIGURE 0x46
-/* CINH register offsets */
-#define QBMAN_CINH_SWP_EQCR_PI 0x800
-#define QBMAN_CINH_SWP_EQCR_CI 0x840
-#define QBMAN_CINH_SWP_EQAR 0x8c0
-#define QBMAN_CINH_SWP_DQPI 0xa00
-#define QBMAN_CINH_SWP_DCAP 0xac0
-#define QBMAN_CINH_SWP_SDQCR 0xb00
-#define QBMAN_CINH_SWP_RAR 0xcc0
-#define QBMAN_CINH_SWP_ISR 0xe00
-#define QBMAN_CINH_SWP_IER 0xe40
-#define QBMAN_CINH_SWP_ISDR 0xe80
-#define QBMAN_CINH_SWP_IIR 0xec0
-
-/* CENA register offsets */
-#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6))
-#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6))
-#define QBMAN_CENA_SWP_RCR(n) (0x400 + ((uint32_t)(n) << 6))
-#define QBMAN_CENA_SWP_CR 0x600
-#define QBMAN_CENA_SWP_RR(vb) (0x700 + ((uint32_t)(vb) >> 1))
-#define QBMAN_CENA_SWP_VDQCR 0x780
-#define QBMAN_CENA_SWP_EQCR_CI 0x840
-
/* Reverse mapping of QBMAN_CENA_SWP_DQRR() */
#define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)p & 0x1ff) >> 6)
/* Pre-defined attribute codes */
/*******************************/
-struct qb_attr_code code_generic_verb = QB_CODE(0, 0, 7);
-struct qb_attr_code code_generic_rslt = QB_CODE(0, 8, 8);
+#define QBMAN_RESPONSE_VERB_MASK 0x7f
/*************************/
/* SDQCR attribute codes */
/*************************/
+#define QB_SDQCR_FC_SHIFT 29
+#define QB_SDQCR_FC_MASK 0x1
+#define QB_SDQCR_DCT_SHIFT 24
+#define QB_SDQCR_DCT_MASK 0x3
+#define QB_SDQCR_TOK_SHIFT 16
+#define QB_SDQCR_TOK_MASK 0xff
+#define QB_SDQCR_SRC_SHIFT 0
+#define QB_SDQCR_SRC_MASK 0xffff
+
+/* opaque token for static dequeues */
+#define QMAN_SDQCR_TOKEN 0xbb
-/* we put these here because at least some of them are required by
- * qbman_swp_init()
- */
-struct qb_attr_code code_sdqcr_dct = QB_CODE(0, 24, 2);
-struct qb_attr_code code_sdqcr_fc = QB_CODE(0, 29, 1);
-struct qb_attr_code code_sdqcr_tok = QB_CODE(0, 16, 8);
-static struct qb_attr_code code_eq_dca_idx;
-#define CODE_SDQCR_DQSRC(n) QB_CODE(0, n, 1)
enum qbman_sdqcr_dct {
qbman_sdqcr_dct_null = 0,
qbman_sdqcr_dct_prio_ics,
qbman_sdqcr_fc_up_to_3 = 1
};
-struct qb_attr_code code_sdqcr_dqsrc = QB_CODE(0, 0, 16);
-
/* We need to keep track of which SWP triggered a pull command
* so keep an array of portal IDs and use the token field to
* be able to find the proper portal
*/
-#define MAX_QBMAN_PORTALS 35
+#define MAX_QBMAN_PORTALS 64
static struct qbman_swp *portal_idx_map[MAX_QBMAN_PORTALS];
-uint32_t qman_version;
+/* Internal Function declaration */
+static int
+qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd);
+static int
+qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd);
+
+static int
+qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd);
+static int
+qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd);
+
+static int
+qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ uint32_t *flags,
+ int num_frames);
+static int
+qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ uint32_t *flags,
+ int num_frames);
+
+static int
+qbman_swp_enqueue_multiple_fd_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ struct qbman_fd **fd,
+ uint32_t *flags,
+ int num_frames);
+
+static int
+qbman_swp_enqueue_multiple_fd_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ struct qbman_fd **fd,
+ uint32_t *flags,
+ int num_frames);
+
+static int
+qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ int num_frames);
+static int
+qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ int num_frames);
+
+static int
+qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d);
+static int
+qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d);
+
+const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s);
+const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s);
+
+static int
+qbman_swp_release_direct(struct qbman_swp *s,
+ const struct qbman_release_desc *d,
+ const uint64_t *buffers, unsigned int num_buffers);
+static int
+qbman_swp_release_mem_back(struct qbman_swp *s,
+ const struct qbman_release_desc *d,
+ const uint64_t *buffers, unsigned int num_buffers);
+
+/* Function pointers */
+static int (*qbman_swp_enqueue_array_mode_ptr)(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd)
+ = qbman_swp_enqueue_array_mode_direct;
+
+static int (*qbman_swp_enqueue_ring_mode_ptr)(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd)
+ = qbman_swp_enqueue_ring_mode_direct;
+
+static int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ uint32_t *flags,
+ int num_frames)
+ = qbman_swp_enqueue_multiple_direct;
+
+static int (*qbman_swp_enqueue_multiple_fd_ptr)(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ struct qbman_fd **fd,
+ uint32_t *flags,
+ int num_frames)
+ = qbman_swp_enqueue_multiple_fd_direct;
+
+static int (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ int num_frames)
+ = qbman_swp_enqueue_multiple_desc_direct;
+
+static int (*qbman_swp_pull_ptr)(struct qbman_swp *s,
+ struct qbman_pull_desc *d)
+ = qbman_swp_pull_direct;
+
+const struct qbman_result *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s)
+ = qbman_swp_dqrr_next_direct;
+
+static int (*qbman_swp_release_ptr)(struct qbman_swp *s,
+ const struct qbman_release_desc *d,
+ const uint64_t *buffers, unsigned int num_buffers)
+ = qbman_swp_release_direct;
/*********************************/
/* Portal constructor/destructor */
{
int ret;
uint32_t eqcr_pi;
- struct qbman_swp *p = kmalloc(sizeof(*p), GFP_KERNEL);
+ uint32_t mask_size;
+ struct qbman_swp *p = malloc(sizeof(*p));
if (!p)
return NULL;
+
+ memset(p, 0, sizeof(struct qbman_swp));
+
p->desc = *d;
#ifdef QBMAN_CHECKING
p->mc.check = swp_mc_can_start;
#endif
p->mc.valid_bit = QB_VALID_BIT;
- p->sdq = 0;
- qb_attr_code_encode(&code_sdqcr_dct, &p->sdq, qbman_sdqcr_dct_prio_ics);
- qb_attr_code_encode(&code_sdqcr_fc, &p->sdq, qbman_sdqcr_fc_up_to_3);
- qb_attr_code_encode(&code_sdqcr_tok, &p->sdq, 0xbb);
+ p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
+ p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
+ p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
+ if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
+ && (d->cena_access_mode == qman_cena_fastest_access))
+ p->mr.valid_bit = QB_VALID_BIT;
+
atomic_set(&p->vdq.busy, 1);
p->vdq.valid_bit = QB_VALID_BIT;
- p->dqrr.next_idx = 0;
p->dqrr.valid_bit = QB_VALID_BIT;
qman_version = p->desc.qman_version;
- if ((qman_version & 0xFFFF0000) < QMAN_REV_4100) {
+ if ((qman_version & QMAN_REV_MASK) < QMAN_REV_4100) {
p->dqrr.dqrr_size = 4;
p->dqrr.reset_bug = 1;
- /* Set size of DQRR to 4, encoded in 2 bits */
- code_eq_dca_idx = (struct qb_attr_code)QB_CODE(0, 8, 2);
} else {
p->dqrr.dqrr_size = 8;
p->dqrr.reset_bug = 0;
- /* Set size of DQRR to 8, encoded in 3 bits */
- code_eq_dca_idx = (struct qb_attr_code)QB_CODE(0, 8, 3);
}
ret = qbman_swp_sys_init(&p->sys, d, p->dqrr.dqrr_size);
if (ret) {
- kfree(p);
+ free(p);
pr_err("qbman_swp_sys_init() failed %d\n", ret);
return NULL;
}
+
+ /* Verify that the DQRRPI is 0 - if it is not the portal isn't
+ * in default state which is an error
+ */
+ if (qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQPI) & 0xF) {
+ pr_err("qbman DQRR PI is not zero, portal is not clean\n");
+ free(p);
+ return NULL;
+ }
+
/* SDQCR needs to be initialized to 0 when no channels are
* being dequeued from or else the QMan HW will indicate an
* error. The values that were calculated above will be
- * applied when dequeues from a specific channel are enabled
+ * applied when dequeues from a specific channel are enabled.
*/
qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_SDQCR, 0);
+
+ p->eqcr.pi_ring_size = 8;
+ if ((qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
+ && (d->cena_access_mode == qman_cena_fastest_access)) {
+ p->eqcr.pi_ring_size = 32;
+ qbman_swp_enqueue_array_mode_ptr =
+ qbman_swp_enqueue_array_mode_mem_back;
+ qbman_swp_enqueue_ring_mode_ptr =
+ qbman_swp_enqueue_ring_mode_mem_back;
+ qbman_swp_enqueue_multiple_ptr =
+ qbman_swp_enqueue_multiple_mem_back;
+ qbman_swp_enqueue_multiple_fd_ptr =
+ qbman_swp_enqueue_multiple_fd_mem_back;
+ qbman_swp_enqueue_multiple_desc_ptr =
+ qbman_swp_enqueue_multiple_desc_mem_back;
+ qbman_swp_pull_ptr = qbman_swp_pull_mem_back;
+ qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back;
+ qbman_swp_release_ptr = qbman_swp_release_mem_back;
+ }
+
+ for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
+ p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask<<1) + 1;
eqcr_pi = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI);
- p->eqcr.pi = eqcr_pi & 0xF;
+ p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT;
- p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_CI) & 0xF;
- p->eqcr.available = QBMAN_EQCR_SIZE - qm_cyc_diff(QBMAN_EQCR_SIZE,
- p->eqcr.ci, p->eqcr.pi);
+ if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
+ && (d->cena_access_mode == qman_cena_fastest_access))
+ p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI)
+ & p->eqcr.pi_ci_mask;
+ else
+ p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_CI)
+ & p->eqcr.pi_ci_mask;
+ p->eqcr.available = p->eqcr.pi_ring_size -
+ qm_cyc_diff(p->eqcr.pi_ring_size,
+ p->eqcr.ci & (p->eqcr.pi_ci_mask<<1),
+ p->eqcr.pi & (p->eqcr.pi_ci_mask<<1));
portal_idx_map[p->desc.idx] = p;
return p;
#endif
qbman_swp_sys_finish(&p->sys);
portal_idx_map[p->desc.idx] = NULL;
- kfree(p);
+ free(p);
}
const struct qbman_swp_desc *qbman_swp_get_desc(struct qbman_swp *p)
qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISR, mask);
}
+uint32_t qbman_swp_dqrr_thrshld_read_status(struct qbman_swp *p)
+{
+ return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQRR_ITR);
+}
+
+void qbman_swp_dqrr_thrshld_write(struct qbman_swp *p, uint32_t mask)
+{
+ qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_DQRR_ITR, mask);
+}
+
+uint32_t qbman_swp_intr_timeout_read_status(struct qbman_swp *p)
+{
+ return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ITPR);
+}
+
+void qbman_swp_intr_timeout_write(struct qbman_swp *p, uint32_t mask)
+{
+ qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ITPR, mask);
+}
+
uint32_t qbman_swp_interrupt_get_trigger(struct qbman_swp *p)
{
return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IER);
void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit)
{
- qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0);
+ qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR,
+ inhibit ? 0xffffffff : 0);
}
/***********************/
#ifdef QBMAN_CHECKING
QBMAN_BUG_ON(p->mc.check != swp_mc_can_start);
#endif
- ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR);
+ if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
+ && (p->desc.cena_access_mode == qman_cena_fastest_access))
+ ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR_MEM);
+ else
+ ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR);
#ifdef QBMAN_CHECKING
if (!ret)
p->mc.check = swp_mc_can_submit;
return ret;
}
-void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint32_t cmd_verb)
+void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb)
{
- uint32_t *v = cmd;
+ uint8_t *v = cmd;
#ifdef QBMAN_CHECKING
QBMAN_BUG_ON(!(p->mc.check != swp_mc_can_submit));
#endif
* caller wants to OR but has forgotten to do so.
*/
QBMAN_BUG_ON((*v & cmd_verb) != *v);
- *v = cmd_verb | p->mc.valid_bit;
- qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd);
+ if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
+ && (p->desc.cena_access_mode == qman_cena_fastest_access)) {
+ *v = cmd_verb | p->mr.valid_bit;
+ qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR_MEM, cmd);
+ dma_wmb();
+ qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE);
+ } else {
+ dma_wmb();
+ *v = cmd_verb | p->mc.valid_bit;
+ qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd);
+ clean(cmd);
+ }
#ifdef QBMAN_CHECKING
p->mc.check = swp_mc_can_poll;
#endif
#ifdef QBMAN_CHECKING
QBMAN_BUG_ON(p->mc.check != swp_mc_can_poll);
#endif
- qbman_cena_invalidate_prefetch(&p->sys,
- QBMAN_CENA_SWP_RR(p->mc.valid_bit));
- ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR(p->mc.valid_bit));
- /* Remove the valid-bit - command completed iff the rest is non-zero */
- verb = ret[0] & ~QB_VALID_BIT;
- if (!verb)
- return NULL;
+ if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
+ && (p->desc.cena_access_mode == qman_cena_fastest_access)) {
+ ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR_MEM);
+ /* Command completed if the valid bit is toggled */
+ if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT))
+ return NULL;
+ /* Remove the valid-bit -
+ * command completed iff the rest is non-zero
+ */
+ verb = ret[0] & ~QB_VALID_BIT;
+ if (!verb)
+ return NULL;
+ p->mr.valid_bit ^= QB_VALID_BIT;
+ } else {
+ qbman_cena_invalidate_prefetch(&p->sys,
+ QBMAN_CENA_SWP_RR(p->mc.valid_bit));
+ ret = qbman_cena_read(&p->sys,
+ QBMAN_CENA_SWP_RR(p->mc.valid_bit));
+ /* Remove the valid-bit -
+ * command completed iff the rest is non-zero
+ */
+ verb = ret[0] & ~QB_VALID_BIT;
+ if (!verb)
+ return NULL;
+ p->mc.valid_bit ^= QB_VALID_BIT;
+ }
#ifdef QBMAN_CHECKING
p->mc.check = swp_mc_can_start;
#endif
- p->mc.valid_bit ^= QB_VALID_BIT;
return ret;
}
/* Enqueue */
/***********/
-/* These should be const, eventually */
-static struct qb_attr_code code_eq_cmd = QB_CODE(0, 0, 2);
-static struct qb_attr_code code_eq_eqdi = QB_CODE(0, 3, 1);
-static struct qb_attr_code code_eq_dca_en = QB_CODE(0, 15, 1);
-static struct qb_attr_code code_eq_dca_pk = QB_CODE(0, 14, 1);
-/* Can't set code_eq_dca_idx width. Need qman version. Read at runtime */
-static struct qb_attr_code code_eq_orp_en = QB_CODE(0, 2, 1);
-static struct qb_attr_code code_eq_orp_is_nesn = QB_CODE(0, 31, 1);
-static struct qb_attr_code code_eq_orp_nlis = QB_CODE(0, 30, 1);
-static struct qb_attr_code code_eq_orp_seqnum = QB_CODE(0, 16, 14);
-static struct qb_attr_code code_eq_opr_id = QB_CODE(1, 0, 16);
-static struct qb_attr_code code_eq_tgt_id = QB_CODE(2, 0, 24);
-/* static struct qb_attr_code code_eq_tag = QB_CODE(3, 0, 32); */
-static struct qb_attr_code code_eq_qd_en = QB_CODE(0, 4, 1);
-static struct qb_attr_code code_eq_qd_bin = QB_CODE(4, 0, 16);
-static struct qb_attr_code code_eq_qd_pri = QB_CODE(4, 16, 4);
-static struct qb_attr_code code_eq_rsp_stash = QB_CODE(5, 16, 1);
-static struct qb_attr_code code_eq_rsp_id = QB_CODE(5, 24, 8);
-static struct qb_attr_code code_eq_rsp_lo = QB_CODE(6, 0, 32);
-
-enum qbman_eq_cmd_e {
- /* No enqueue, primarily for plugging ORP gaps for dropped frames */
- qbman_eq_cmd_empty,
- /* DMA an enqueue response once complete */
- qbman_eq_cmd_respond,
- /* DMA an enqueue response only if the enqueue fails */
- qbman_eq_cmd_respond_reject
+#define QB_ENQUEUE_CMD_OPTIONS_SHIFT 0
+enum qb_enqueue_commands {
+ enqueue_empty = 0,
+ enqueue_response_always = 1,
+ enqueue_rejects_to_fq = 2
};
+#define QB_ENQUEUE_CMD_EC_OPTION_MASK 0x3
+#define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT 2
+#define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3
+#define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT 4
+#define QB_ENQUEUE_CMD_DCA_PK_SHIFT 6
+#define QB_ENQUEUE_CMD_DCA_EN_SHIFT 7
+#define QB_ENQUEUE_CMD_NLIS_SHIFT 14
+#define QB_ENQUEUE_CMD_IS_NESN_SHIFT 15
+
void qbman_eq_desc_clear(struct qbman_eq_desc *d)
{
memset(d, 0, sizeof(*d));
void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_eq_orp_en, cl, 0);
- qb_attr_code_encode(&code_eq_cmd, cl,
- respond_success ? qbman_eq_cmd_respond :
- qbman_eq_cmd_respond_reject);
+ d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT);
+ if (respond_success)
+ d->eq.verb |= enqueue_response_always;
+ else
+ d->eq.verb |= enqueue_rejects_to_fq;
}
void qbman_eq_desc_set_orp(struct qbman_eq_desc *d, int respond_success,
- uint32_t opr_id, uint32_t seqnum, int incomplete)
+ uint16_t opr_id, uint16_t seqnum, int incomplete)
{
- uint32_t *cl = qb_cl(d);
+ d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT;
+ if (respond_success)
+ d->eq.verb |= enqueue_response_always;
+ else
+ d->eq.verb |= enqueue_rejects_to_fq;
- qb_attr_code_encode(&code_eq_orp_en, cl, 1);
- qb_attr_code_encode(&code_eq_cmd, cl,
- respond_success ? qbman_eq_cmd_respond :
- qbman_eq_cmd_respond_reject);
- qb_attr_code_encode(&code_eq_opr_id, cl, opr_id);
- qb_attr_code_encode(&code_eq_orp_seqnum, cl, seqnum);
- qb_attr_code_encode(&code_eq_orp_nlis, cl, !!incomplete);
+ d->eq.orpid = opr_id;
+ d->eq.seqnum = seqnum;
+ if (incomplete)
+ d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_NLIS_SHIFT;
+ else
+ d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT);
}
-void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint32_t opr_id,
- uint32_t seqnum)
+void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint16_t opr_id,
+ uint16_t seqnum)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_eq_orp_en, cl, 1);
- qb_attr_code_encode(&code_eq_cmd, cl, qbman_eq_cmd_empty);
- qb_attr_code_encode(&code_eq_opr_id, cl, opr_id);
- qb_attr_code_encode(&code_eq_orp_seqnum, cl, seqnum);
- qb_attr_code_encode(&code_eq_orp_nlis, cl, 0);
- qb_attr_code_encode(&code_eq_orp_is_nesn, cl, 0);
+ d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT;
+ d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK;
+ d->eq.orpid = opr_id;
+ d->eq.seqnum = seqnum;
+ d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT);
+ d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT);
}
-void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint32_t opr_id,
- uint32_t seqnum)
+void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint16_t opr_id,
+ uint16_t seqnum)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_eq_orp_en, cl, 1);
- qb_attr_code_encode(&code_eq_cmd, cl, qbman_eq_cmd_empty);
- qb_attr_code_encode(&code_eq_opr_id, cl, opr_id);
- qb_attr_code_encode(&code_eq_orp_seqnum, cl, seqnum);
- qb_attr_code_encode(&code_eq_orp_nlis, cl, 0);
- qb_attr_code_encode(&code_eq_orp_is_nesn, cl, 1);
+ d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT;
+ d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK;
+ d->eq.orpid = opr_id;
+ d->eq.seqnum = seqnum;
+ d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT);
+ d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT;
}
void qbman_eq_desc_set_response(struct qbman_eq_desc *d,
dma_addr_t storage_phys,
int stash)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode_64(&code_eq_rsp_lo, (uint64_t *)cl, storage_phys);
- qb_attr_code_encode(&code_eq_rsp_stash, cl, !!stash);
+ d->eq.rsp_addr = storage_phys;
+ d->eq.wae = stash;
}
void qbman_eq_desc_set_token(struct qbman_eq_desc *d, uint8_t token)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_eq_rsp_id, cl, (uint32_t)token);
+ d->eq.rspid = token;
}
void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, uint32_t fqid)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_eq_qd_en, cl, 0);
- qb_attr_code_encode(&code_eq_tgt_id, cl, fqid);
+ d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT);
+ d->eq.tgtid = fqid;
}
void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, uint32_t qdid,
- uint32_t qd_bin, uint32_t qd_prio)
+ uint16_t qd_bin, uint8_t qd_prio)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_eq_qd_en, cl, 1);
- qb_attr_code_encode(&code_eq_tgt_id, cl, qdid);
- qb_attr_code_encode(&code_eq_qd_bin, cl, qd_bin);
- qb_attr_code_encode(&code_eq_qd_pri, cl, qd_prio);
+ d->eq.verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT;
+ d->eq.tgtid = qdid;
+ d->eq.qdbin = qd_bin;
+ d->eq.qpri = qd_prio;
}
void qbman_eq_desc_set_eqdi(struct qbman_eq_desc *d, int enable)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_eq_eqdi, cl, !!enable);
+ if (enable)
+ d->eq.verb |= 1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT;
+ else
+ d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT);
}
void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable,
- uint32_t dqrr_idx, int park)
+ uint8_t dqrr_idx, int park)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_eq_dca_en, cl, !!enable);
if (enable) {
- qb_attr_code_encode(&code_eq_dca_pk, cl, !!park);
- qb_attr_code_encode(&code_eq_dca_idx, cl, dqrr_idx);
+ d->eq.dca = dqrr_idx;
+ if (park)
+ d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT;
+ else
+ d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT);
+ d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT;
+ } else {
+ d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT);
}
}
-#define EQAR_IDX(eqar) ((eqar) & 0x7)
+#define EQAR_IDX(eqar) ((eqar) & 0x1f)
#define EQAR_VB(eqar) ((eqar) & 0x80)
#define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
-static int qbman_swp_enqueue_array_mode(struct qbman_swp *s,
- const struct qbman_eq_desc *d,
- const struct qbman_fd *fd)
+
+static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p,
+ uint8_t idx)
+{
+ if (idx < 16)
+ qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4,
+ QMAN_RT_MODE);
+ else
+ qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT2 +
+ (idx - 16) * 4,
+ QMAN_RT_MODE);
+}
+
+
+static int qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd)
{
uint32_t *p;
const uint32_t *cl = qb_cl(d);
return -EBUSY;
p = qbman_cena_write_start_wo_shadow(&s->sys,
QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
- word_copy(&p[1], &cl[1], 7);
- word_copy(&p[8], fd, sizeof(*fd) >> 2);
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], fd, sizeof(*fd));
+
/* Set the verb byte, have to substitute in the valid-bit */
- lwsync();
+ dma_wmb();
p[0] = cl[0] | EQAR_VB(eqar);
qbman_cena_write_complete_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
+ return 0;
+}
+static int qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd)
+{
+ uint32_t *p;
+ const uint32_t *cl = qb_cl(d);
+ uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR);
+
+ pr_debug("EQAR=%08x\n", eqar);
+ if (!EQAR_SUCCESS(eqar))
+ return -EBUSY;
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], fd, sizeof(*fd));
+
+ /* Set the verb byte, have to substitute in the valid-bit */
+ p[0] = cl[0] | EQAR_VB(eqar);
+ dma_wmb();
+ qbman_write_eqcr_am_rt_register(s, EQAR_IDX(eqar));
return 0;
}
-static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s,
- const struct qbman_eq_desc *d,
- const struct qbman_fd *fd)
+static inline int qbman_swp_enqueue_array_mode(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd)
+{
+ return qbman_swp_enqueue_array_mode_ptr(s, d, fd);
+}
+
+static int qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd)
{
uint32_t *p;
const uint32_t *cl = qb_cl(d);
- uint32_t eqcr_ci;
- uint8_t diff;
+ uint32_t eqcr_ci, full_mask, half_mask;
+ half_mask = (s->eqcr.pi_ci_mask>>1);
+ full_mask = s->eqcr.pi_ci_mask;
if (!s->eqcr.available) {
eqcr_ci = s->eqcr.ci;
s->eqcr.ci = qbman_cena_read_reg(&s->sys,
- QBMAN_CENA_SWP_EQCR_CI) & 0xF;
- diff = qm_cyc_diff(QBMAN_EQCR_SIZE,
- eqcr_ci, s->eqcr.ci);
- s->eqcr.available += diff;
- if (!diff)
+ QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+ s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+ eqcr_ci, s->eqcr.ci);
+ if (!s->eqcr.available)
return -EBUSY;
}
p = qbman_cena_write_start_wo_shadow(&s->sys,
- QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7));
- word_copy(&p[1], &cl[1], 7);
- word_copy(&p[8], fd, sizeof(*fd) >> 2);
+ QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], fd, sizeof(*fd));
lwsync();
+
/* Set the verb byte, have to substitute in the valid-bit */
p[0] = cl[0] | s->eqcr.pi_vb;
qbman_cena_write_complete_wo_shadow(&s->sys,
- QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7));
+ QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
s->eqcr.pi++;
- s->eqcr.pi &= 0xF;
+ s->eqcr.pi &= full_mask;
s->eqcr.available--;
- if (!(s->eqcr.pi & 7))
+ if (!(s->eqcr.pi & half_mask))
s->eqcr.pi_vb ^= QB_VALID_BIT;
+
return 0;
}
-int qbman_swp_fill_ring(struct qbman_swp *s,
- const struct qbman_eq_desc *d,
- const struct qbman_fd *fd,
- __attribute__((unused)) uint8_t burst_index)
+static int qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd)
{
uint32_t *p;
const uint32_t *cl = qb_cl(d);
- uint32_t eqcr_ci;
- uint8_t diff;
+ uint32_t eqcr_ci, full_mask, half_mask;
+ half_mask = (s->eqcr.pi_ci_mask>>1);
+ full_mask = s->eqcr.pi_ci_mask;
if (!s->eqcr.available) {
eqcr_ci = s->eqcr.ci;
s->eqcr.ci = qbman_cena_read_reg(&s->sys,
- QBMAN_CENA_SWP_EQCR_CI) & 0xF;
- diff = qm_cyc_diff(QBMAN_EQCR_SIZE,
- eqcr_ci, s->eqcr.ci);
- s->eqcr.available += diff;
- if (!diff)
+ QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask;
+ s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+ eqcr_ci, s->eqcr.ci);
+ if (!s->eqcr.available)
return -EBUSY;
}
+
p = qbman_cena_write_start_wo_shadow(&s->sys,
- QBMAN_CENA_SWP_EQCR((s->eqcr.pi/* +burst_index */) & 7));
- /* word_copy(&p[1], &cl[1], 7); */
- memcpy(&p[1], &cl[1], 7 * 4);
- /* word_copy(&p[8], fd, sizeof(*fd) >> 2); */
- memcpy(&p[8], fd, sizeof(struct qbman_fd));
+ QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], fd, sizeof(*fd));
- /* lwsync(); */
+ /* Set the verb byte, have to substitute in the valid-bit */
p[0] = cl[0] | s->eqcr.pi_vb;
-
s->eqcr.pi++;
- s->eqcr.pi &= 0xF;
+ s->eqcr.pi &= full_mask;
s->eqcr.available--;
- if (!(s->eqcr.pi & 7))
+ if (!(s->eqcr.pi & half_mask))
s->eqcr.pi_vb ^= QB_VALID_BIT;
-
+ dma_wmb();
+ qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
+ (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
return 0;
}
-int qbman_swp_flush_ring(struct qbman_swp *s)
-{
- void *ptr = s->sys.addr_cena;
-
- dcbf((uint64_t)ptr);
- dcbf((uint64_t)ptr + 0x40);
- dcbf((uint64_t)ptr + 0x80);
- dcbf((uint64_t)ptr + 0xc0);
- dcbf((uint64_t)ptr + 0x100);
- dcbf((uint64_t)ptr + 0x140);
- dcbf((uint64_t)ptr + 0x180);
- dcbf((uint64_t)ptr + 0x1c0);
-
- return 0;
-}
-
-void qbman_sync(void)
+static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd)
{
- lwsync();
+ return qbman_swp_enqueue_ring_mode_ptr(s, d, fd);
}
int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d,
return qbman_swp_enqueue_ring_mode(s, d, fd);
}
+static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ uint32_t *flags,
+ int num_frames)
+{
+ uint32_t *p = NULL;
+ const uint32_t *cl = qb_cl(d);
+ uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
+ int i, num_enqueued = 0;
+ uint64_t addr_cena;
+
+ half_mask = (s->eqcr.pi_ci_mask>>1);
+ full_mask = s->eqcr.pi_ci_mask;
+ if (!s->eqcr.available) {
+ eqcr_ci = s->eqcr.ci;
+ s->eqcr.ci = qbman_cena_read_reg(&s->sys,
+ QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+ s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+ eqcr_ci, s->eqcr.ci);
+ if (!s->eqcr.available)
+ return 0;
+ }
+
+ eqcr_pi = s->eqcr.pi;
+ num_enqueued = (s->eqcr.available < num_frames) ?
+ s->eqcr.available : num_frames;
+ s->eqcr.available -= num_enqueued;
+ /* Fill in the EQCR ring */
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], &fd[i], sizeof(*fd));
+ eqcr_pi++;
+ }
+
+ lwsync();
+
+ /* Set the verb byte, have to substitute in the valid-bit */
+ eqcr_pi = s->eqcr.pi;
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ p[0] = cl[0] | s->eqcr.pi_vb;
+ if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
+ struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
+
+ d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
+ ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
+ }
+ eqcr_pi++;
+ if (!(eqcr_pi & half_mask))
+ s->eqcr.pi_vb ^= QB_VALID_BIT;
+ }
+
+ /* Flush all the cacheline without load/store in between */
+ eqcr_pi = s->eqcr.pi;
+ addr_cena = (size_t)s->sys.addr_cena;
+ for (i = 0; i < num_enqueued; i++) {
+ dcbf((uintptr_t)(addr_cena +
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)));
+ eqcr_pi++;
+ }
+ s->eqcr.pi = eqcr_pi & full_mask;
+
+ return num_enqueued;
+}
+
+static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ uint32_t *flags,
+ int num_frames)
+{
+ uint32_t *p = NULL;
+ const uint32_t *cl = qb_cl(d);
+ uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
+ int i, num_enqueued = 0;
+
+ half_mask = (s->eqcr.pi_ci_mask>>1);
+ full_mask = s->eqcr.pi_ci_mask;
+ if (!s->eqcr.available) {
+ eqcr_ci = s->eqcr.ci;
+ s->eqcr.ci = qbman_cena_read_reg(&s->sys,
+ QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask;
+ s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+ eqcr_ci, s->eqcr.ci);
+ if (!s->eqcr.available)
+ return 0;
+ }
+
+ eqcr_pi = s->eqcr.pi;
+ num_enqueued = (s->eqcr.available < num_frames) ?
+ s->eqcr.available : num_frames;
+ s->eqcr.available -= num_enqueued;
+ /* Fill in the EQCR ring */
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], &fd[i], sizeof(*fd));
+ if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
+ struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
+
+ d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
+ ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
+ }
+ eqcr_pi++;
+ p[0] = cl[0] | s->eqcr.pi_vb;
+
+ if (!(eqcr_pi & half_mask))
+ s->eqcr.pi_vb ^= QB_VALID_BIT;
+ }
+ s->eqcr.pi = eqcr_pi & full_mask;
+
+ dma_wmb();
+ qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
+ (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
+ return num_enqueued;
+}
+
+inline int qbman_swp_enqueue_multiple(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ uint32_t *flags,
+ int num_frames)
+{
+ return qbman_swp_enqueue_multiple_ptr(s, d, fd, flags, num_frames);
+}
+
+static int qbman_swp_enqueue_multiple_fd_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ struct qbman_fd **fd,
+ uint32_t *flags,
+ int num_frames)
+{
+ uint32_t *p = NULL;
+ const uint32_t *cl = qb_cl(d);
+ uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
+ int i, num_enqueued = 0;
+ uint64_t addr_cena;
+
+ half_mask = (s->eqcr.pi_ci_mask>>1);
+ full_mask = s->eqcr.pi_ci_mask;
+ if (!s->eqcr.available) {
+ eqcr_ci = s->eqcr.ci;
+ s->eqcr.ci = qbman_cena_read_reg(&s->sys,
+ QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+ s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+ eqcr_ci, s->eqcr.ci);
+ if (!s->eqcr.available)
+ return 0;
+ }
+
+ eqcr_pi = s->eqcr.pi;
+ num_enqueued = (s->eqcr.available < num_frames) ?
+ s->eqcr.available : num_frames;
+ s->eqcr.available -= num_enqueued;
+ /* Fill in the EQCR ring */
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], fd[i], sizeof(struct qbman_fd));
+ eqcr_pi++;
+ }
+
+ lwsync();
+
+ /* Set the verb byte, have to substitute in the valid-bit */
+ eqcr_pi = s->eqcr.pi;
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ p[0] = cl[0] | s->eqcr.pi_vb;
+ if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
+ struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
+
+ d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
+ ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
+ }
+ eqcr_pi++;
+ if (!(eqcr_pi & half_mask))
+ s->eqcr.pi_vb ^= QB_VALID_BIT;
+ }
+
+ /* Flush all the cacheline without load/store in between */
+ eqcr_pi = s->eqcr.pi;
+ addr_cena = (size_t)s->sys.addr_cena;
+ for (i = 0; i < num_enqueued; i++) {
+ dcbf(addr_cena +
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ eqcr_pi++;
+ }
+ s->eqcr.pi = eqcr_pi & full_mask;
+
+ return num_enqueued;
+}
+
+static int qbman_swp_enqueue_multiple_fd_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ struct qbman_fd **fd,
+ uint32_t *flags,
+ int num_frames)
+{
+ uint32_t *p = NULL;
+ const uint32_t *cl = qb_cl(d);
+ uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
+ int i, num_enqueued = 0;
+
+ half_mask = (s->eqcr.pi_ci_mask>>1);
+ full_mask = s->eqcr.pi_ci_mask;
+ if (!s->eqcr.available) {
+ eqcr_ci = s->eqcr.ci;
+ s->eqcr.ci = qbman_cena_read_reg(&s->sys,
+ QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask;
+ s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+ eqcr_ci, s->eqcr.ci);
+ if (!s->eqcr.available)
+ return 0;
+ }
+
+ eqcr_pi = s->eqcr.pi;
+ num_enqueued = (s->eqcr.available < num_frames) ?
+ s->eqcr.available : num_frames;
+ s->eqcr.available -= num_enqueued;
+ /* Fill in the EQCR ring */
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], fd[i], sizeof(struct qbman_fd));
+ eqcr_pi++;
+ }
+
+ /* Set the verb byte, have to substitute in the valid-bit */
+ eqcr_pi = s->eqcr.pi;
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ p[0] = cl[0] | s->eqcr.pi_vb;
+ if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
+ struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
+
+ d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
+ ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
+ }
+ eqcr_pi++;
+ if (!(eqcr_pi & half_mask))
+ s->eqcr.pi_vb ^= QB_VALID_BIT;
+ }
+ s->eqcr.pi = eqcr_pi & full_mask;
+
+ dma_wmb();
+ qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
+ (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
+ return num_enqueued;
+}
+
+inline int qbman_swp_enqueue_multiple_fd(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ struct qbman_fd **fd,
+ uint32_t *flags,
+ int num_frames)
+{
+ return qbman_swp_enqueue_multiple_fd_ptr(s, d, fd, flags, num_frames);
+}
+
+static int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ int num_frames)
+{
+ uint32_t *p;
+ const uint32_t *cl;
+ uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
+ int i, num_enqueued = 0;
+ uint64_t addr_cena;
+
+ half_mask = (s->eqcr.pi_ci_mask>>1);
+ full_mask = s->eqcr.pi_ci_mask;
+ if (!s->eqcr.available) {
+ eqcr_ci = s->eqcr.ci;
+ s->eqcr.ci = qbman_cena_read_reg(&s->sys,
+ QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+ s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+ eqcr_ci, s->eqcr.ci);
+ if (!s->eqcr.available)
+ return 0;
+ }
+
+ eqcr_pi = s->eqcr.pi;
+ num_enqueued = (s->eqcr.available < num_frames) ?
+ s->eqcr.available : num_frames;
+ s->eqcr.available -= num_enqueued;
+ /* Fill in the EQCR ring */
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ cl = qb_cl(&d[i]);
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], &fd[i], sizeof(*fd));
+ eqcr_pi++;
+ }
+
+ lwsync();
+
+ /* Set the verb byte, have to substitute in the valid-bit */
+ eqcr_pi = s->eqcr.pi;
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ cl = qb_cl(&d[i]);
+ p[0] = cl[0] | s->eqcr.pi_vb;
+ eqcr_pi++;
+ if (!(eqcr_pi & half_mask))
+ s->eqcr.pi_vb ^= QB_VALID_BIT;
+ }
+
+ /* Flush all the cacheline without load/store in between */
+ eqcr_pi = s->eqcr.pi;
+ addr_cena = (size_t)s->sys.addr_cena;
+ for (i = 0; i < num_enqueued; i++) {
+ dcbf((uintptr_t)(addr_cena +
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)));
+ eqcr_pi++;
+ }
+ s->eqcr.pi = eqcr_pi & full_mask;
+
+ return num_enqueued;
+}
+
+static int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ int num_frames)
+{
+ uint32_t *p;
+ const uint32_t *cl;
+ uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
+ int i, num_enqueued = 0;
+
+ half_mask = (s->eqcr.pi_ci_mask>>1);
+ full_mask = s->eqcr.pi_ci_mask;
+ if (!s->eqcr.available) {
+ eqcr_ci = s->eqcr.ci;
+ s->eqcr.ci = qbman_cena_read_reg(&s->sys,
+ QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask;
+ s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+ eqcr_ci, s->eqcr.ci);
+ if (!s->eqcr.available)
+ return 0;
+ }
+
+ eqcr_pi = s->eqcr.pi;
+ num_enqueued = (s->eqcr.available < num_frames) ?
+ s->eqcr.available : num_frames;
+ s->eqcr.available -= num_enqueued;
+ /* Fill in the EQCR ring */
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ cl = qb_cl(&d[i]);
+ memcpy(&p[1], &cl[1], 28);
+ memcpy(&p[8], &fd[i], sizeof(*fd));
+ eqcr_pi++;
+ }
+
+ /* Set the verb byte, have to substitute in the valid-bit */
+ eqcr_pi = s->eqcr.pi;
+ for (i = 0; i < num_enqueued; i++) {
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+ cl = qb_cl(&d[i]);
+ p[0] = cl[0] | s->eqcr.pi_vb;
+ eqcr_pi++;
+ if (!(eqcr_pi & half_mask))
+ s->eqcr.pi_vb ^= QB_VALID_BIT;
+ }
+
+ s->eqcr.pi = eqcr_pi & full_mask;
+
+ dma_wmb();
+ qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
+ (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
+
+ return num_enqueued;
+}
+inline int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
+ const struct qbman_eq_desc *d,
+ const struct qbman_fd *fd,
+ int num_frames)
+{
+ return qbman_swp_enqueue_multiple_desc_ptr(s, d, fd, num_frames);
+}
+
/*************************/
/* Static (push) dequeue */
/*************************/
void qbman_swp_push_get(struct qbman_swp *s, uint8_t channel_idx, int *enabled)
{
- struct qb_attr_code code = CODE_SDQCR_DQSRC(channel_idx);
+ uint16_t src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
QBMAN_BUG_ON(channel_idx > 15);
- *enabled = (int)qb_attr_code_decode(&code, &s->sdq);
+ *enabled = src | (1 << channel_idx);
}
void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable)
{
uint16_t dqsrc;
- struct qb_attr_code code = CODE_SDQCR_DQSRC(channel_idx);
QBMAN_BUG_ON(channel_idx > 15);
- qb_attr_code_encode(&code, &s->sdq, !!enable);
+ if (enable)
+ s->sdq |= 1 << channel_idx;
+ else
+ s->sdq &= ~(1 << channel_idx);
+
/* Read make the complete src map. If no channels are enabled
* the SDQCR must be 0 or else QMan will assert errors
*/
- dqsrc = (uint16_t)qb_attr_code_decode(&code_sdqcr_dqsrc, &s->sdq);
+ dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
if (dqsrc != 0)
qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, s->sdq);
else
/***************************/
/* These should be const, eventually */
-static struct qb_attr_code code_pull_dct = QB_CODE(0, 0, 2);
-static struct qb_attr_code code_pull_dt = QB_CODE(0, 2, 2);
-static struct qb_attr_code code_pull_rls = QB_CODE(0, 4, 1);
-static struct qb_attr_code code_pull_stash = QB_CODE(0, 5, 1);
-static struct qb_attr_code code_pull_numframes = QB_CODE(0, 8, 4);
-static struct qb_attr_code code_pull_token = QB_CODE(0, 16, 8);
-static struct qb_attr_code code_pull_dqsource = QB_CODE(1, 0, 24);
-static struct qb_attr_code code_pull_rsp_lo = QB_CODE(2, 0, 32);
+#define QB_VDQCR_VERB_DCT_SHIFT 0
+#define QB_VDQCR_VERB_DT_SHIFT 2
+#define QB_VDQCR_VERB_RLS_SHIFT 4
+#define QB_VDQCR_VERB_WAE_SHIFT 5
+#define QB_VDQCR_VERB_RAD_SHIFT 6
enum qb_pull_dt_e {
qb_pull_dt_channel,
dma_addr_t storage_phys,
int stash)
{
- uint32_t *cl = qb_cl(d);
- /* Squiggle the pointer 'storage' into the extra 2 words of the
- * descriptor (which aren't copied to the hw command)
- */
- *(void **)&cl[4] = storage;
+ d->pull.rsp_addr_virt = (size_t)storage;
+
if (!storage) {
- qb_attr_code_encode(&code_pull_rls, cl, 0);
+ d->pull.verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT);
return;
}
- qb_attr_code_encode(&code_pull_rls, cl, 1);
- qb_attr_code_encode(&code_pull_stash, cl, !!stash);
- qb_attr_code_encode_64(&code_pull_rsp_lo, (uint64_t *)cl, storage_phys);
+ d->pull.verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT;
+ if (stash)
+ d->pull.verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT;
+ else
+ d->pull.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
+
+ d->pull.rsp_addr = storage_phys;
}
-void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, uint8_t numframes)
+void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d,
+ uint8_t numframes)
{
- uint32_t *cl = qb_cl(d);
-
- QBMAN_BUG_ON(!numframes || (numframes > 16));
- qb_attr_code_encode(&code_pull_numframes, cl,
- (uint32_t)(numframes - 1));
+ d->pull.numf = numframes - 1;
}
void qbman_pull_desc_set_token(struct qbman_pull_desc *d, uint8_t token)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_pull_token, cl, token);
+ d->pull.tok = token;
}
void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, uint32_t fqid)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_pull_dct, cl, 1);
- qb_attr_code_encode(&code_pull_dt, cl, qb_pull_dt_framequeue);
- qb_attr_code_encode(&code_pull_dqsource, cl, fqid);
+ d->pull.verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT;
+ d->pull.verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT;
+ d->pull.dq_src = fqid;
}
void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, uint32_t wqid,
enum qbman_pull_type_e dct)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_pull_dct, cl, dct);
- qb_attr_code_encode(&code_pull_dt, cl, qb_pull_dt_workqueue);
- qb_attr_code_encode(&code_pull_dqsource, cl, wqid);
+ d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
+ d->pull.verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT;
+ d->pull.dq_src = wqid;
}
void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid,
enum qbman_pull_type_e dct)
{
- uint32_t *cl = qb_cl(d);
+ d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
+ d->pull.verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT;
+ d->pull.dq_src = chid;
+}
- qb_attr_code_encode(&code_pull_dct, cl, dct);
- qb_attr_code_encode(&code_pull_dt, cl, qb_pull_dt_channel);
- qb_attr_code_encode(&code_pull_dqsource, cl, chid);
+void qbman_pull_desc_set_rad(struct qbman_pull_desc *d, int rad)
+{
+ if (d->pull.verb & (1 << QB_VDQCR_VERB_RLS_SHIFT)) {
+ if (rad)
+ d->pull.verb |= 1 << QB_VDQCR_VERB_RAD_SHIFT;
+ else
+ d->pull.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT);
+ } else {
+ printf("The RAD feature is not valid when RLS = 0\n");
+ }
}
-int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
+static int qbman_swp_pull_direct(struct qbman_swp *s,
+ struct qbman_pull_desc *d)
{
uint32_t *p;
uint32_t *cl = qb_cl(d);
atomic_inc(&s->vdq.busy);
return -EBUSY;
}
- s->vdq.storage = *(void **)&cl[4];
- /* We use portal index +1 as token so that 0 still indicates
- * that the result isn't valid yet.
- */
- qb_attr_code_encode(&code_pull_token, cl, s->desc.idx + 1);
+
+ d->pull.tok = s->sys.idx + 1;
+ s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt;
p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR);
- word_copy(&p[1], &cl[1], 3);
+ memcpy(&p[1], &cl[1], 12);
+
/* Set the verb byte, have to substitute in the valid-bit */
lwsync();
p[0] = cl[0] | s->vdq.valid_bit;
s->vdq.valid_bit ^= QB_VALID_BIT;
qbman_cena_write_complete_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR);
+
return 0;
}
+static int qbman_swp_pull_mem_back(struct qbman_swp *s,
+ struct qbman_pull_desc *d)
+{
+ uint32_t *p;
+ uint32_t *cl = qb_cl(d);
+
+ if (!atomic_dec_and_test(&s->vdq.busy)) {
+ atomic_inc(&s->vdq.busy);
+ return -EBUSY;
+ }
+
+ d->pull.tok = s->sys.idx + 1;
+ s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt;
+ p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR_MEM);
+ memcpy(&p[1], &cl[1], 12);
+
+ /* Set the verb byte, have to substitute in the valid-bit */
+ p[0] = cl[0] | s->vdq.valid_bit;
+ s->vdq.valid_bit ^= QB_VALID_BIT;
+ dma_wmb();
+ qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE);
+
+ return 0;
+}
+
+inline int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
+{
+ return qbman_swp_pull_ptr(s, d);
+}
+
/****************/
/* Polling DQRR */
/****************/
-static struct qb_attr_code code_dqrr_verb = QB_CODE(0, 0, 8);
-static struct qb_attr_code code_dqrr_response = QB_CODE(0, 0, 7);
-static struct qb_attr_code code_dqrr_stat = QB_CODE(0, 8, 8);
-static struct qb_attr_code code_dqrr_seqnum = QB_CODE(0, 16, 14);
-static struct qb_attr_code code_dqrr_odpid = QB_CODE(1, 0, 16);
-/* static struct qb_attr_code code_dqrr_tok = QB_CODE(1, 24, 8); */
-static struct qb_attr_code code_dqrr_fqid = QB_CODE(2, 0, 24);
-static struct qb_attr_code code_dqrr_byte_count = QB_CODE(4, 0, 32);
-static struct qb_attr_code code_dqrr_frame_count = QB_CODE(5, 0, 24);
-static struct qb_attr_code code_dqrr_ctx_lo = QB_CODE(6, 0, 32);
+#define QMAN_DQRR_PI_MASK 0xf
#define QBMAN_RESULT_DQ 0x60
#define QBMAN_RESULT_FQRN 0x21
#define QBMAN_RESULT_BPSCN 0x29
#define QBMAN_RESULT_CSCN_WQ 0x2a
-static struct qb_attr_code code_dqpi_pi = QB_CODE(0, 0, 4);
+#include <rte_prefetch.h>
+
+void qbman_swp_prefetch_dqrr_next(struct qbman_swp *s)
+{
+ const struct qbman_result *p;
+
+ p = qbman_cena_read_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
+ rte_prefetch0(p);
+}
/* NULL return if there are no unconsumed DQRR entries. Returns a DQRR entry
* only once, so repeated calls can return a sequence of DQRR entries, without
* requiring they be consumed immediately or in any particular order.
*/
-const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s)
+inline const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s)
+{
+ return qbman_swp_dqrr_next_ptr(s);
+}
+
+const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s)
{
uint32_t verb;
uint32_t response_verb;
uint32_t flags;
- const struct qbman_result *dq;
- const uint32_t *p;
+ const struct qbman_result *p;
/* Before using valid-bit to detect if something is there, we have to
* handle the case of the DQRR reset bug...
*/
- if (unlikely(s->dqrr.reset_bug)) {
+ if (s->dqrr.reset_bug) {
/* We pick up new entries by cache-inhibited producer index,
* which means that a non-coherent mapping would require us to
* invalidate and read *only* once that PI has indicated that
* will be much less efficient than all subsequent trips around
* it...
*/
- uint32_t dqpi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI);
- uint32_t pi = qb_attr_code_decode(&code_dqpi_pi, &dqpi);
- /* there are new entries iff pi != next_idx */
+ uint8_t pi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI) &
+ QMAN_DQRR_PI_MASK;
+
+ /* there are new entries if pi != next_idx */
if (pi == s->dqrr.next_idx)
return NULL;
+
/* if next_idx is/was the last ring index, and 'pi' is
* different, we can disable the workaround as all the ring
* entries have now been DMA'd to so valid-bit checking is
s->dqrr.reset_bug = 0;
}
qbman_cena_invalidate_prefetch(&s->sys,
- QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
+ QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
}
- dq = qbman_cena_read_wo_shadow(&s->sys,
- QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
- p = qb_cl(dq);
- verb = qb_attr_code_decode(&code_dqrr_verb, p);
+ p = qbman_cena_read_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
+
+ verb = p->dq.verb;
+
/* If the valid-bit isn't of the expected polarity, nothing there. Note,
* in the DQRR reset bug workaround, we shouldn't need to skip these
* check, because we've already determined that a new entry is available
s->dqrr.valid_bit ^= QB_VALID_BIT;
}
/* If this is the final response to a volatile dequeue command
- * indicate that the vdq is no longer busy.
+ * indicate that the vdq is no longer busy
*/
- flags = qbman_result_DQ_flags(dq);
- response_verb = qb_attr_code_decode(&code_dqrr_response, &verb);
+ flags = p->dq.stat;
+ response_verb = verb & QBMAN_RESPONSE_VERB_MASK;
if ((response_verb == QBMAN_RESULT_DQ) &&
(flags & QBMAN_DQ_STAT_VOLATILE) &&
(flags & QBMAN_DQ_STAT_EXPIRED))
atomic_inc(&s->vdq.busy);
- return dq;
+ return p;
+}
+
+const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s)
+{
+ uint32_t verb;
+ uint32_t response_verb;
+ uint32_t flags;
+ const struct qbman_result *p;
+
+ p = qbman_cena_read_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx));
+
+ verb = p->dq.verb;
+
+ /* If the valid-bit isn't of the expected polarity, nothing there. Note,
+ * in the DQRR reset bug workaround, we shouldn't need to skip these
+ * check, because we've already determined that a new entry is available
+ * and we've invalidated the cacheline before reading it, so the
+ * valid-bit behaviour is repaired and should tell us what we already
+ * knew from reading PI.
+ */
+ if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit)
+ return NULL;
+
+ /* There's something there. Move "next_idx" attention to the next ring
+ * entry (and prefetch it) before returning what we found.
+ */
+ s->dqrr.next_idx++;
+ if (s->dqrr.next_idx == s->dqrr.dqrr_size) {
+ s->dqrr.next_idx = 0;
+ s->dqrr.valid_bit ^= QB_VALID_BIT;
+ }
+ /* If this is the final response to a volatile dequeue command
+ * indicate that the vdq is no longer busy
+ */
+ flags = p->dq.stat;
+ response_verb = verb & QBMAN_RESPONSE_VERB_MASK;
+ if ((response_verb == QBMAN_RESULT_DQ)
+ && (flags & QBMAN_DQ_STAT_VOLATILE)
+ && (flags & QBMAN_DQ_STAT_EXPIRED))
+ atomic_inc(&s->vdq.busy);
+ return p;
}
/* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */
void qbman_swp_dqrr_consume(struct qbman_swp *s,
const struct qbman_result *dq)
{
- qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
+ qbman_cinh_write(&s->sys,
+ QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
+}
+
+/* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */
+void qbman_swp_dqrr_idx_consume(struct qbman_swp *s,
+ uint8_t dqrr_index)
+{
+ qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, dqrr_index);
}
/*********************************/
/* Polling user-provided storage */
/*********************************/
-int qbman_result_has_new_result(__attribute__((unused)) struct qbman_swp *s,
- const struct qbman_result *dq)
+int qbman_result_has_new_result(struct qbman_swp *s,
+ struct qbman_result *dq)
{
- /* To avoid converting the little-endian DQ entry to host-endian prior
- * to us knowing whether there is a valid entry or not (and run the
- * risk of corrupting the incoming hardware LE write), we detect in
- * hardware endianness rather than host. This means we need a different
- * "code" depending on whether we are BE or LE in software, which is
- * where DQRR_TOK_OFFSET comes in...
+ if (dq->dq.tok == 0)
+ return 0;
+
+ /*
+ * Set token to be 0 so we will detect change back to 1
+ * next time the looping is traversed. Const is cast away here
+ * as we want users to treat the dequeue responses as read only.
*/
- static struct qb_attr_code code_dqrr_tok_detect =
- QB_CODE(0, DQRR_TOK_OFFSET, 8);
- /* The user trying to poll for a result treats "dq" as const. It is
- * however the same address that was provided to us non-const in the
- * first place, for directing hardware DMA to. So we can cast away the
- * const because it is mutable from our perspective.
+ ((struct qbman_result *)dq)->dq.tok = 0;
+
+ /*
+ * VDQCR "no longer busy" hook - not quite the same as DQRR, because
+ * the fact "VDQCR" shows busy doesn't mean that we hold the result
+ * that makes it available. Eg. we may be looking at our 10th dequeue
+ * result, having released VDQCR after the 1st result and it is now
+ * busy due to some other command!
*/
- uint32_t *p = (uint32_t *)(unsigned long)qb_cl(dq);
- uint32_t token;
+ if (s->vdq.storage == dq) {
+ s->vdq.storage = NULL;
+ atomic_inc(&s->vdq.busy);
+ }
+
+ return 1;
+}
- token = qb_attr_code_decode(&code_dqrr_tok_detect, &p[1]);
- if (token == 0)
+int qbman_check_new_result(struct qbman_result *dq)
+{
+ if (dq->dq.tok == 0)
return 0;
- /* Entry is valid - overwrite token back to 0 so
- * a) If this memory is reused tokesn will be 0
- * b) If someone calls "has_new_result()" again on this entry it
- * will not appear to be new
- */
- qb_attr_code_encode(&code_dqrr_tok_detect, &p[1], 0);
- /* Only now do we convert from hardware to host endianness. Also, as we
- * are returning success, the user has promised not to call us again, so
- * there's no risk of us converting the endianness twice...
+ /*
+ * Set token to be 0 so we will detect change back to 1
+ * next time the looping is traversed. Const is cast away here
+ * as we want users to treat the dequeue responses as read only.
*/
- make_le32_n(p, 16);
+ ((struct qbman_result *)dq)->dq.tok = 0;
+
return 1;
}
-int qbman_check_command_complete(struct qbman_swp *s,
- const struct qbman_result *dq)
+int qbman_check_command_complete(struct qbman_result *dq)
{
- /* To avoid converting the little-endian DQ entry to host-endian prior
- * to us knowing whether there is a valid entry or not (and run the
- * risk of corrupting the incoming hardware LE write), we detect in
- * hardware endianness rather than host. This means we need a different
- * "code" depending on whether we are BE or LE in software, which is
- * where DQRR_TOK_OFFSET comes in...
- */
- static struct qb_attr_code code_dqrr_tok_detect =
- QB_CODE(0, DQRR_TOK_OFFSET, 8);
- /* The user trying to poll for a result treats "dq" as const. It is
- * however the same address that was provided to us non-const in the
- * first place, for directing hardware DMA to. So we can cast away the
- * const because it is mutable from our perspective.
- */
- uint32_t *p = (uint32_t *)(unsigned long)qb_cl(dq);
- uint32_t token;
+ struct qbman_swp *s;
- token = qb_attr_code_decode(&code_dqrr_tok_detect, &p[1]);
- if (token == 0)
+ if (dq->dq.tok == 0)
return 0;
- /* TODO: Remove qbman_swp from parameters and make it a local
- * once we've tested the reserve portal map change
- */
- s = portal_idx_map[token - 1];
- /* When token is set it indicates that VDQ command has been fetched
- * by qbman and is working on it. It is safe for software to issue
- * another VDQ command, so incrementing the busy variable.
+
+ s = portal_idx_map[dq->dq.tok - 1];
+ /*
+ * VDQCR "no longer busy" hook - not quite the same as DQRR, because
+ * the fact "VDQCR" shows busy doesn't mean that we hold the result
+ * that makes it available. Eg. we may be looking at our 10th dequeue
+ * result, having released VDQCR after the 1st result and it is now
+ * busy due to some other command!
*/
if (s->vdq.storage == dq) {
s->vdq.storage = NULL;
atomic_inc(&s->vdq.busy);
}
+
return 1;
}
/* Categorising qbman results */
/********************************/
-static struct qb_attr_code code_result_in_mem =
- QB_CODE(0, QBMAN_RESULT_VERB_OFFSET_IN_MEM, 7);
-
static inline int __qbman_result_is_x(const struct qbman_result *dq,
- uint32_t x)
-{
- const uint32_t *p = qb_cl(dq);
- uint32_t response_verb = qb_attr_code_decode(&code_dqrr_response, p);
-
- return (response_verb == x);
-}
-
-static inline int __qbman_result_is_x_in_mem(const struct qbman_result *dq,
- uint32_t x)
+ uint8_t x)
{
- const uint32_t *p = qb_cl(dq);
- uint32_t response_verb = qb_attr_code_decode(&code_result_in_mem, p);
+ uint8_t response_verb = dq->dq.verb & QBMAN_RESPONSE_VERB_MASK;
return (response_verb == x);
}
int qbman_result_is_CSCN(const struct qbman_result *dq)
{
- return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_CSCN_MEM) ||
+ return __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_MEM) ||
__qbman_result_is_x(dq, QBMAN_RESULT_CSCN_WQ);
}
int qbman_result_is_BPSCN(const struct qbman_result *dq)
{
- return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_BPSCN);
+ return __qbman_result_is_x(dq, QBMAN_RESULT_BPSCN);
}
int qbman_result_is_CGCU(const struct qbman_result *dq)
{
- return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_CGCU);
+ return __qbman_result_is_x(dq, QBMAN_RESULT_CGCU);
}
int qbman_result_is_FQRN(const struct qbman_result *dq)
{
- return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_FQRN);
+ return __qbman_result_is_x(dq, QBMAN_RESULT_FQRN);
}
int qbman_result_is_FQRNI(const struct qbman_result *dq)
{
- return __qbman_result_is_x_in_mem(dq, QBMAN_RESULT_FQRNI);
+ return __qbman_result_is_x(dq, QBMAN_RESULT_FQRNI);
}
int qbman_result_is_FQPN(const struct qbman_result *dq)
/* These APIs assume qbman_result_is_DQ() is TRUE */
-uint32_t qbman_result_DQ_flags(const struct qbman_result *dq)
+uint8_t qbman_result_DQ_flags(const struct qbman_result *dq)
{
- const uint32_t *p = qb_cl(dq);
-
- return qb_attr_code_decode(&code_dqrr_stat, p);
+ return dq->dq.stat;
}
uint16_t qbman_result_DQ_seqnum(const struct qbman_result *dq)
{
- const uint32_t *p = qb_cl(dq);
-
- return (uint16_t)qb_attr_code_decode(&code_dqrr_seqnum, p);
+ return dq->dq.seqnum;
}
uint16_t qbman_result_DQ_odpid(const struct qbman_result *dq)
{
- const uint32_t *p = qb_cl(dq);
-
- return (uint16_t)qb_attr_code_decode(&code_dqrr_odpid, p);
+ return dq->dq.oprid;
}
uint32_t qbman_result_DQ_fqid(const struct qbman_result *dq)
{
- const uint32_t *p = qb_cl(dq);
-
- return qb_attr_code_decode(&code_dqrr_fqid, p);
+ return dq->dq.fqid;
}
uint32_t qbman_result_DQ_byte_count(const struct qbman_result *dq)
{
- const uint32_t *p = qb_cl(dq);
-
- return qb_attr_code_decode(&code_dqrr_byte_count, p);
+ return dq->dq.fq_byte_cnt;
}
uint32_t qbman_result_DQ_frame_count(const struct qbman_result *dq)
{
- const uint32_t *p = qb_cl(dq);
-
- return qb_attr_code_decode(&code_dqrr_frame_count, p);
+ return dq->dq.fq_frm_cnt;
}
uint64_t qbman_result_DQ_fqd_ctx(const struct qbman_result *dq)
{
- const uint64_t *p = (const uint64_t *)qb_cl(dq);
-
- return qb_attr_code_decode_64(&code_dqrr_ctx_lo, p);
+ return dq->dq.fqd_ctx;
}
const struct qbman_fd *qbman_result_DQ_fd(const struct qbman_result *dq)
{
- const uint32_t *p = qb_cl(dq);
-
- return (const struct qbman_fd *)&p[8];
+ return (const struct qbman_fd *)&dq->dq.fd[0];
}
/**************************************/
/* Parsing state-change notifications */
/**************************************/
-
-static struct qb_attr_code code_scn_state = QB_CODE(0, 16, 8);
-static struct qb_attr_code code_scn_rid = QB_CODE(1, 0, 24);
-static struct qb_attr_code code_scn_state_in_mem =
- QB_CODE(0, SCN_STATE_OFFSET_IN_MEM, 8);
-static struct qb_attr_code code_scn_rid_in_mem =
- QB_CODE(1, SCN_RID_OFFSET_IN_MEM, 24);
-static struct qb_attr_code code_scn_ctx_lo = QB_CODE(2, 0, 32);
-
uint8_t qbman_result_SCN_state(const struct qbman_result *scn)
{
- const uint32_t *p = qb_cl(scn);
-
- return (uint8_t)qb_attr_code_decode(&code_scn_state, p);
+ return scn->scn.state;
}
uint32_t qbman_result_SCN_rid(const struct qbman_result *scn)
{
- const uint32_t *p = qb_cl(scn);
-
- return qb_attr_code_decode(&code_scn_rid, p);
+ return scn->scn.rid_tok;
}
uint64_t qbman_result_SCN_ctx(const struct qbman_result *scn)
{
- const uint64_t *p = (const uint64_t *)qb_cl(scn);
-
- return qb_attr_code_decode_64(&code_scn_ctx_lo, p);
-}
-
-uint8_t qbman_result_SCN_state_in_mem(const struct qbman_result *scn)
-{
- const uint32_t *p = qb_cl(scn);
-
- return (uint8_t)qb_attr_code_decode(&code_scn_state_in_mem, p);
-}
-
-uint32_t qbman_result_SCN_rid_in_mem(const struct qbman_result *scn)
-{
- const uint32_t *p = qb_cl(scn);
- uint32_t result_rid;
-
- result_rid = qb_attr_code_decode(&code_scn_rid_in_mem, p);
- return make_le24(result_rid);
+ return scn->scn.ctx;
}
/*****************/
/*****************/
uint16_t qbman_result_bpscn_bpid(const struct qbman_result *scn)
{
- return (uint16_t)qbman_result_SCN_rid_in_mem(scn) & 0x3FFF;
+ return (uint16_t)qbman_result_SCN_rid(scn) & 0x3FFF;
}
int qbman_result_bpscn_has_free_bufs(const struct qbman_result *scn)
{
- return !(int)(qbman_result_SCN_state_in_mem(scn) & 0x1);
+ return !(int)(qbman_result_SCN_state(scn) & 0x1);
}
int qbman_result_bpscn_is_depleted(const struct qbman_result *scn)
{
- return (int)(qbman_result_SCN_state_in_mem(scn) & 0x2);
+ return (int)(qbman_result_SCN_state(scn) & 0x2);
}
int qbman_result_bpscn_is_surplus(const struct qbman_result *scn)
{
- return (int)(qbman_result_SCN_state_in_mem(scn) & 0x4);
+ return (int)(qbman_result_SCN_state(scn) & 0x4);
}
uint64_t qbman_result_bpscn_ctx(const struct qbman_result *scn)
{
- uint64_t ctx;
- uint32_t ctx_hi, ctx_lo;
-
- ctx = qbman_result_SCN_ctx(scn);
- ctx_hi = upper32(ctx);
- ctx_lo = lower32(ctx);
- return ((uint64_t)make_le32(ctx_hi) << 32 |
- (uint64_t)make_le32(ctx_lo));
+ return qbman_result_SCN_ctx(scn);
}
/*****************/
/*****************/
uint16_t qbman_result_cgcu_cgid(const struct qbman_result *scn)
{
- return (uint16_t)qbman_result_SCN_rid_in_mem(scn) & 0xFFFF;
+ return (uint16_t)qbman_result_SCN_rid(scn) & 0xFFFF;
}
uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn)
{
- uint64_t ctx;
- uint32_t ctx_hi, ctx_lo;
+ return qbman_result_SCN_ctx(scn);
+}
+
+/********************/
+/* Parsing EQ RESP */
+/********************/
+struct qbman_fd *qbman_result_eqresp_fd(struct qbman_result *eqresp)
+{
+ return (struct qbman_fd *)&eqresp->eq_resp.fd[0];
+}
- ctx = qbman_result_SCN_ctx(scn);
- ctx_hi = upper32(ctx);
- ctx_lo = lower32(ctx);
- return ((uint64_t)(make_le32(ctx_hi) & 0xFF) << 32) |
- (uint64_t)make_le32(ctx_lo);
+void qbman_result_eqresp_set_rspid(struct qbman_result *eqresp, uint8_t val)
+{
+ eqresp->eq_resp.rspid = val;
+}
+
+uint8_t qbman_result_eqresp_rspid(struct qbman_result *eqresp)
+{
+ return eqresp->eq_resp.rspid;
+}
+
+uint8_t qbman_result_eqresp_rc(struct qbman_result *eqresp)
+{
+ if (eqresp->eq_resp.rc == 0xE)
+ return 0;
+ else
+ return -1;
}
/******************/
/* Buffer release */
/******************/
-
-/* These should be const, eventually */
-/* static struct qb_attr_code code_release_num = QB_CODE(0, 0, 3); */
-static struct qb_attr_code code_release_set_me = QB_CODE(0, 5, 1);
-static struct qb_attr_code code_release_rcdi = QB_CODE(0, 6, 1);
-static struct qb_attr_code code_release_bpid = QB_CODE(0, 16, 16);
+#define QB_BR_RC_VALID_SHIFT 5
+#define QB_BR_RCDI_SHIFT 6
void qbman_release_desc_clear(struct qbman_release_desc *d)
{
- uint32_t *cl;
-
memset(d, 0, sizeof(*d));
- cl = qb_cl(d);
- qb_attr_code_encode(&code_release_set_me, cl, 1);
+ d->br.verb = 1 << QB_BR_RC_VALID_SHIFT;
}
-void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint32_t bpid)
+void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint16_t bpid)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_release_bpid, cl, bpid);
+ d->br.bpid = bpid;
}
void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
{
- uint32_t *cl = qb_cl(d);
-
- qb_attr_code_encode(&code_release_rcdi, cl, !!enable);
+ if (enable)
+ d->br.verb |= 1 << QB_BR_RCDI_SHIFT;
+ else
+ d->br.verb &= ~(1 << QB_BR_RCDI_SHIFT);
}
#define RAR_IDX(rar) ((rar) & 0x7)
#define RAR_VB(rar) ((rar) & 0x80)
#define RAR_SUCCESS(rar) ((rar) & 0x100)
-int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d,
- const uint64_t *buffers, unsigned int num_buffers)
+static int qbman_swp_release_direct(struct qbman_swp *s,
+ const struct qbman_release_desc *d,
+ const uint64_t *buffers,
+ unsigned int num_buffers)
{
uint32_t *p;
const uint32_t *cl = qb_cl(d);
pr_debug("RAR=%08x\n", rar);
if (!RAR_SUCCESS(rar))
return -EBUSY;
+
QBMAN_BUG_ON(!num_buffers || (num_buffers > 7));
+
/* Start the release command */
p = qbman_cena_write_start_wo_shadow(&s->sys,
- QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
+ QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
+
/* Copy the caller's buffer pointers to the command */
u64_to_le32_copy(&p[2], buffers, num_buffers);
- /* Set the verb byte, have to substitute in the valid-bit and the number
- * of buffers.
+
+ /* Set the verb byte, have to substitute in the valid-bit and the
+ * number of buffers.
*/
lwsync();
p[0] = cl[0] | RAR_VB(rar) | num_buffers;
qbman_cena_write_complete_wo_shadow(&s->sys,
- QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
+ QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
+
+ return 0;
+}
+
+static int qbman_swp_release_mem_back(struct qbman_swp *s,
+ const struct qbman_release_desc *d,
+ const uint64_t *buffers,
+ unsigned int num_buffers)
+{
+ uint32_t *p;
+ const uint32_t *cl = qb_cl(d);
+ uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR);
+
+ pr_debug("RAR=%08x\n", rar);
+ if (!RAR_SUCCESS(rar))
+ return -EBUSY;
+
+ QBMAN_BUG_ON(!num_buffers || (num_buffers > 7));
+
+ /* Start the release command */
+ p = qbman_cena_write_start_wo_shadow(&s->sys,
+ QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar)));
+
+ /* Copy the caller's buffer pointers to the command */
+ u64_to_le32_copy(&p[2], buffers, num_buffers);
+
+ /* Set the verb byte, have to substitute in the valid-bit and the
+ * number of buffers.
+ */
+ p[0] = cl[0] | RAR_VB(rar) | num_buffers;
+ lwsync();
+ qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_RCR_AM_RT +
+ RAR_IDX(rar) * 4, QMAN_RT_MODE);
+
return 0;
}
+inline int qbman_swp_release(struct qbman_swp *s,
+ const struct qbman_release_desc *d,
+ const uint64_t *buffers,
+ unsigned int num_buffers)
+{
+ return qbman_swp_release_ptr(s, d, buffers, num_buffers);
+}
+
/*******************/
/* Buffer acquires */
/*******************/
+struct qbman_acquire_desc {
+ uint8_t verb;
+ uint8_t reserved;
+ uint16_t bpid;
+ uint8_t num;
+ uint8_t reserved2[59];
+};
-/* These should be const, eventually */
-static struct qb_attr_code code_acquire_bpid = QB_CODE(0, 16, 16);
-static struct qb_attr_code code_acquire_num = QB_CODE(1, 0, 3);
-static struct qb_attr_code code_acquire_r_num = QB_CODE(1, 0, 3);
+struct qbman_acquire_rslt {
+ uint8_t verb;
+ uint8_t rslt;
+ uint16_t reserved;
+ uint8_t num;
+ uint8_t reserved2[3];
+ uint64_t buf[7];
+};
-int qbman_swp_acquire(struct qbman_swp *s, uint32_t bpid, uint64_t *buffers,
+int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers,
unsigned int num_buffers)
{
- uint32_t *p;
- uint32_t rslt, num;
+ struct qbman_acquire_desc *p;
+ struct qbman_acquire_rslt *r;
- QBMAN_BUG_ON(!num_buffers || (num_buffers > 7));
+ if (!num_buffers || (num_buffers > 7))
+ return -EINVAL;
/* Start the management command */
p = qbman_swp_mc_start(s);
return -EBUSY;
/* Encode the caller-provided attributes */
- qb_attr_code_encode(&code_acquire_bpid, p, bpid);
- qb_attr_code_encode(&code_acquire_num, p, num_buffers);
+ p->bpid = bpid;
+ p->num = num_buffers;
/* Complete the management command */
- p = qbman_swp_mc_complete(s, p, p[0] | QBMAN_MC_ACQUIRE);
+ r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE);
+ if (!r) {
+ pr_err("qbman: acquire from BPID %d failed, no response\n",
+ bpid);
+ return -EIO;
+ }
/* Decode the outcome */
- rslt = qb_attr_code_decode(&code_generic_rslt, p);
- num = qb_attr_code_decode(&code_acquire_r_num, p);
- QBMAN_BUG_ON(qb_attr_code_decode(&code_generic_verb, p) !=
- QBMAN_MC_ACQUIRE);
+ QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE);
/* Determine success or failure */
- if (unlikely(rslt != QBMAN_MC_RSLT_OK)) {
+ if (r->rslt != QBMAN_MC_RSLT_OK) {
pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n",
- bpid, rslt);
+ bpid, r->rslt);
return -EIO;
}
- QBMAN_BUG_ON(num > num_buffers);
+
+ QBMAN_BUG_ON(r->num > num_buffers);
+
/* Copy the acquired buffers to the caller's array */
- u64_from_le32_copy(buffers, &p[2], num);
- return (int)num;
+ u64_from_le32_copy(buffers, &r->buf[0], r->num);
+
+ return (int)r->num;
}
/*****************/
/* FQ management */
/*****************/
+struct qbman_alt_fq_state_desc {
+ uint8_t verb;
+ uint8_t reserved[3];
+ uint32_t fqid;
+ uint8_t reserved2[56];
+};
+
+struct qbman_alt_fq_state_rslt {
+ uint8_t verb;
+ uint8_t rslt;
+ uint8_t reserved[62];
+};
-static struct qb_attr_code code_fqalt_fqid = QB_CODE(1, 0, 32);
+#define ALT_FQ_FQID_MASK 0x00FFFFFF
static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid,
uint8_t alt_fq_verb)
{
- uint32_t *p;
- uint32_t rslt;
+ struct qbman_alt_fq_state_desc *p;
+ struct qbman_alt_fq_state_rslt *r;
/* Start the management command */
p = qbman_swp_mc_start(s);
if (!p)
return -EBUSY;
- qb_attr_code_encode(&code_fqalt_fqid, p, fqid);
+ p->fqid = fqid & ALT_FQ_FQID_MASK;
+
/* Complete the management command */
- p = qbman_swp_mc_complete(s, p, p[0] | alt_fq_verb);
+ r = qbman_swp_mc_complete(s, p, alt_fq_verb);
+ if (!r) {
+ pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n",
+ alt_fq_verb);
+ return -EIO;
+ }
/* Decode the outcome */
- rslt = qb_attr_code_decode(&code_generic_rslt, p);
- QBMAN_BUG_ON(qb_attr_code_decode(&code_generic_verb, p) != alt_fq_verb);
+ QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != alt_fq_verb);
/* Determine success or failure */
- if (unlikely(rslt != QBMAN_MC_RSLT_OK)) {
+ if (r->rslt != QBMAN_MC_RSLT_OK) {
pr_err("ALT FQID %d failed: verb = 0x%08x, code = 0x%02x\n",
- fqid, alt_fq_verb, rslt);
+ fqid, alt_fq_verb, r->rslt);
return -EIO;
}
/* Channel management */
/**********************/
-static struct qb_attr_code code_cdan_cid = QB_CODE(0, 16, 12);
-static struct qb_attr_code code_cdan_we = QB_CODE(1, 0, 8);
-static struct qb_attr_code code_cdan_en = QB_CODE(1, 8, 1);
-static struct qb_attr_code code_cdan_ctx_lo = QB_CODE(2, 0, 32);
+struct qbman_cdan_ctrl_desc {
+ uint8_t verb;
+ uint8_t reserved;
+ uint16_t ch;
+ uint8_t we;
+ uint8_t ctrl;
+ uint16_t reserved2;
+ uint64_t cdan_ctx;
+ uint8_t reserved3[48];
+
+};
+
+struct qbman_cdan_ctrl_rslt {
+ uint8_t verb;
+ uint8_t rslt;
+ uint16_t ch;
+ uint8_t reserved[60];
+};
/* Hide "ICD" for now as we don't use it, don't set it, and don't test it, so it
* would be irresponsible to expose it.
uint8_t we_mask, uint8_t cdan_en,
uint64_t ctx)
{
- uint32_t *p;
- uint32_t rslt;
+ struct qbman_cdan_ctrl_desc *p;
+ struct qbman_cdan_ctrl_rslt *r;
/* Start the management command */
p = qbman_swp_mc_start(s);
return -EBUSY;
/* Encode the caller-provided attributes */
- qb_attr_code_encode(&code_cdan_cid, p, channelid);
- qb_attr_code_encode(&code_cdan_we, p, we_mask);
- qb_attr_code_encode(&code_cdan_en, p, cdan_en);
- qb_attr_code_encode_64(&code_cdan_ctx_lo, (uint64_t *)p, ctx);
+ p->ch = channelid;
+ p->we = we_mask;
+ if (cdan_en)
+ p->ctrl = 1;
+ else
+ p->ctrl = 0;
+ p->cdan_ctx = ctx;
+
/* Complete the management command */
- p = qbman_swp_mc_complete(s, p, p[0] | QBMAN_WQCHAN_CONFIGURE);
+ r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE);
+ if (!r) {
+ pr_err("qbman: wqchan config failed, no response\n");
+ return -EIO;
+ }
/* Decode the outcome */
- rslt = qb_attr_code_decode(&code_generic_rslt, p);
- QBMAN_BUG_ON(qb_attr_code_decode(&code_generic_verb, p)
- != QBMAN_WQCHAN_CONFIGURE);
+ QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK)
+ != QBMAN_WQCHAN_CONFIGURE);
/* Determine success or failure */
- if (unlikely(rslt != QBMAN_MC_RSLT_OK)) {
+ if (r->rslt != QBMAN_MC_RSLT_OK) {
pr_err("CDAN cQID %d failed: code = 0x%02x\n",
- channelid, rslt);
+ channelid, r->rslt);
return -EIO;
}
1, ctx);
}
-uint8_t qbman_get_dqrr_idx(struct qbman_result *dqrr)
+uint8_t qbman_get_dqrr_idx(const struct qbman_result *dqrr)
{
return QBMAN_IDX_FROM_DQRR(dqrr);
}
dq = qbman_cena_read(&s->sys, QBMAN_CENA_SWP_DQRR(idx));
return dq;
}
-
-int qbman_swp_send_multiple(struct qbman_swp *s,
- const struct qbman_eq_desc *d,
- const struct qbman_fd *fd,
- int frames_to_send)
-{
- uint32_t *p;
- const uint32_t *cl = qb_cl(d);
- uint32_t eqcr_ci;
- uint8_t diff;
- int sent = 0;
- int i;
- int initial_pi = s->eqcr.pi;
- uint64_t start_pointer;
-
- if (!s->eqcr.available) {
- eqcr_ci = s->eqcr.ci;
- s->eqcr.ci = qbman_cena_read_reg(&s->sys,
- QBMAN_CENA_SWP_EQCR_CI) & 0xF;
- diff = qm_cyc_diff(QBMAN_EQCR_SIZE,
- eqcr_ci, s->eqcr.ci);
- if (!diff)
- goto done;
- s->eqcr.available += diff;
- }
-
- /* we are trying to send frames_to_send,
- * if we have enough space in the ring
- */
- while (s->eqcr.available && frames_to_send--) {
- p = qbman_cena_write_start_wo_shadow_fast(&s->sys,
- QBMAN_CENA_SWP_EQCR((initial_pi) & 7));
- /* Write command (except of first byte) and FD */
- memcpy(&p[1], &cl[1], 7 * 4);
- memcpy(&p[8], &fd[sent], sizeof(struct qbman_fd));
-
- initial_pi++;
- initial_pi &= 0xF;
- s->eqcr.available--;
- sent++;
- }
-
-done:
- initial_pi = s->eqcr.pi;
- lwsync();
-
- /* in order for flushes to complete faster:
- * we use a following trick: we record all lines in 32 bit word
- */
-
- initial_pi = s->eqcr.pi;
- for (i = 0; i < sent; i++) {
- p = qbman_cena_write_start_wo_shadow_fast(&s->sys,
- QBMAN_CENA_SWP_EQCR((initial_pi) & 7));
-
- p[0] = cl[0] | s->eqcr.pi_vb;
- initial_pi++;
- initial_pi &= 0xF;
-
- if (!(initial_pi & 7))
- s->eqcr.pi_vb ^= QB_VALID_BIT;
- }
-
- initial_pi = s->eqcr.pi;
-
- /* We need to flush all the lines but without
- * load/store operations between them.
- * We assign start_pointer before we start loop so that
- * in loop we do not read it from memory
- */
- start_pointer = (uint64_t)s->sys.addr_cena;
- for (i = 0; i < sent; i++) {
- p = (uint32_t *)(start_pointer
- + QBMAN_CENA_SWP_EQCR(initial_pi & 7));
- dcbf((uint64_t)p);
- initial_pi++;
- initial_pi &= 0xF;
- }
-
- /* Update producer index for the next call */
- s->eqcr.pi = initial_pi;
-
- return sent;
-}
-
-int qbman_get_version(void)
-{
- return qman_version;
-}
projects / ceph.git / blobdiff