[ceph.git] / ceph / src / spdk / dpdk / drivers / bus / fslmc / qbman / qbman_sys.h

/* SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
 *
 */
/* qbman_sys_decl.h and qbman_sys.h are the two platform-specific files in the
 * driver. They are only included via qbman_private.h, which is itself a
 * platform-independent file and is included by all the other driver source.
 *
 * qbman_sys_decl.h is included prior to all other declarations and logic, and
 * it exists to provide compatibility with any linux interfaces our
 * single-source driver code is dependent on (eg. kmalloc). Ie. this file
 * provides linux compatibility.
 *
 * This qbman_sys.h header, on the other hand, is included *after* any common
 * and platform-neutral declarations and logic in qbman_private.h, and exists to
 * implement any platform-specific logic of the qbman driver itself. Ie. it is
 * *not* to provide linux compatibility.
 */

#ifndef _QBMAN_SYS_H_
#define _QBMAN_SYS_H_

#include "qbman_sys_decl.h"

#define CENA_WRITE_ENABLE 0
#define CINH_WRITE_ENABLE 1

/* CINH register offsets */
#define QBMAN_CINH_SWP_EQCR_PI      0x800
#define QBMAN_CINH_SWP_EQCR_CI      0x840
#define QBMAN_CINH_SWP_EQAR         0x8c0
#define QBMAN_CINH_SWP_CR_RT        0x900
#define QBMAN_CINH_SWP_VDQCR_RT     0x940
#define QBMAN_CINH_SWP_EQCR_AM_RT   0x980
#define QBMAN_CINH_SWP_RCR_AM_RT    0x9c0
#define QBMAN_CINH_SWP_DQPI         0xa00
#define QBMAN_CINH_SWP_DQRR_ITR     0xa80
#define QBMAN_CINH_SWP_DCAP         0xac0
#define QBMAN_CINH_SWP_SDQCR        0xb00
#define QBMAN_CINH_SWP_EQCR_AM_RT2  0xb40
#define QBMAN_CINH_SWP_RCR_PI       0xc00
#define QBMAN_CINH_SWP_RAR          0xcc0
#define QBMAN_CINH_SWP_ISR          0xe00
#define QBMAN_CINH_SWP_IER          0xe40
#define QBMAN_CINH_SWP_ISDR         0xe80
#define QBMAN_CINH_SWP_IIR          0xec0
#define QBMAN_CINH_SWP_ITPR         0xf40

/* CENA register offsets */
#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6))
#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6))
#define QBMAN_CENA_SWP_RCR(n)  (0x400 + ((uint32_t)(n) << 6))
#define QBMAN_CENA_SWP_CR      0x600
#define QBMAN_CENA_SWP_RR(vb)  (0x700 + ((uint32_t)(vb) >> 1))
#define QBMAN_CENA_SWP_VDQCR   0x780
#define QBMAN_CENA_SWP_EQCR_CI 0x840
#define QBMAN_CENA_SWP_EQCR_CI_MEMBACK 0x1840

/* CENA register offsets in memory-backed mode */
#define QBMAN_CENA_SWP_DQRR_MEM(n)  (0x800 + ((uint32_t)(n) << 6))
#define QBMAN_CENA_SWP_RCR_MEM(n)   (0x1400 + ((uint32_t)(n) << 6))
#define QBMAN_CENA_SWP_CR_MEM       0x1600
#define QBMAN_CENA_SWP_RR_MEM       0x1680
#define QBMAN_CENA_SWP_VDQCR_MEM    0x1780

/* Debugging assists */
static inline void __hexdump(unsigned long start, unsigned long end,
			     unsigned long p, size_t sz, const unsigned char *c)
{
	while (start < end) {
		unsigned int pos = 0;
		char buf[64];
		int nl = 0;

		pos += sprintf(buf + pos, "%08lx: ", start);
		do {
			if ((start < p) || (start >= (p + sz)))
				pos += sprintf(buf + pos, "..");
			else
				pos += sprintf(buf + pos, "%02x", *(c++));
			if (!(++start & 15)) {
				buf[pos++] = '\n';
				nl = 1;
			} else {
				nl = 0;
				if (!(start & 1))
					buf[pos++] = ' ';
				if (!(start & 3))
					buf[pos++] = ' ';
			}
		} while (start & 15);
		if (!nl)
			buf[pos++] = '\n';
		buf[pos] = '\0';
		pr_info("%s", buf);
	}
}

static inline void hexdump(const void *ptr, size_t sz)
{
	unsigned long p = (unsigned long)ptr;
	unsigned long start = p & ~15;
	unsigned long end = (p + sz + 15) & ~15;
	const unsigned char *c = ptr;

	__hexdump(start, end, p, sz, c);
}

/* Currently, the CENA support code expects each 32-bit word to be written in
 * host order, and these are converted to hardware (little-endian) order on
 * command submission. However, 64-bit quantities are must be written (and read)
 * as two 32-bit words with the least-significant word first, irrespective of
 * host endianness.
 */
static inline void u64_to_le32_copy(void *d, const uint64_t *s,
				    unsigned int cnt)
{
	uint32_t *dd = d;
	const uint32_t *ss = (const uint32_t *)s;

	while (cnt--) {
		/* TBD: the toolchain was choking on the use of 64-bit types up
		 * until recently so this works entirely with 32-bit variables.
		 * When 64-bit types become usable again, investigate better
		 * ways of doing this.
		 */
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
		*(dd++) = ss[1];
		*(dd++) = ss[0];
		ss += 2;
#else
		*(dd++) = *(ss++);
		*(dd++) = *(ss++);
#endif
	}
}

static inline void u64_from_le32_copy(uint64_t *d, const void *s,
				      unsigned int cnt)
{
	const uint32_t *ss = s;
	uint32_t *dd = (uint32_t *)d;

	while (cnt--) {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
		dd[1] = *(ss++);
		dd[0] = *(ss++);
		dd += 2;
#else
		*(dd++) = *(ss++);
		*(dd++) = *(ss++);
#endif
	}
}

	/******************/
	/* Portal access  */
	/******************/
struct qbman_swp_sys {
	/* On GPP, the sys support for qbman_swp is here. The CENA region isi
	 * not an mmap() of the real portal registers, but an allocated
	 * place-holder, because the actual writes/reads to/from the portal are
	 * marshalled from these allocated areas using QBMan's "MC access
	 * registers". CINH accesses are atomic so there's no need for a
	 * place-holder.
	 */
	uint8_t *cena;
	uint8_t *addr_cena;
	uint8_t *addr_cinh;
	uint32_t idx;
	enum qbman_eqcr_mode eqcr_mode;
};

/* P_OFFSET is (ACCESS_CMD,0,12) - offset within the portal
 * C is (ACCESS_CMD,12,1) - is inhibited? (0==CENA, 1==CINH)
 * SWP_IDX is (ACCESS_CMD,16,10) - Software portal index
 * P is (ACCESS_CMD,28,1) - (0==special portal, 1==any portal)
 * T is (ACCESS_CMD,29,1) - Command type (0==READ, 1==WRITE)
 * E is (ACCESS_CMD,31,1) - Command execute (1 to issue, poll for 0==complete)
 */

static inline void qbman_cinh_write(struct qbman_swp_sys *s, uint32_t offset,
				    uint32_t val)
{
	__raw_writel(val, s->addr_cinh + offset);
#ifdef QBMAN_CINH_TRACE
	pr_info("qbman_cinh_write(%p:%d:0x%03x) 0x%08x\n",
		s->addr_cinh, s->idx, offset, val);
#endif
}

static inline uint32_t qbman_cinh_read(struct qbman_swp_sys *s, uint32_t offset)
{
	uint32_t reg = __raw_readl(s->addr_cinh + offset);
#ifdef QBMAN_CINH_TRACE
	pr_info("qbman_cinh_read(%p:%d:0x%03x) 0x%08x\n",
		s->addr_cinh, s->idx, offset, reg);
#endif
	return reg;
}

static inline void *qbman_cena_write_start(struct qbman_swp_sys *s,
					   uint32_t offset)
{
	void *shadow = s->cena + offset;

#ifdef QBMAN_CENA_TRACE
	pr_info("qbman_cena_write_start(%p:%d:0x%03x) %p\n",
		s->addr_cena, s->idx, offset, shadow);
#endif
	QBMAN_BUG_ON(offset & 63);
	dcbz(shadow);
	return shadow;
}

static inline void *qbman_cena_write_start_wo_shadow(struct qbman_swp_sys *s,
						     uint32_t offset)
{
#ifdef QBMAN_CENA_TRACE
	pr_info("qbman_cena_write_start(%p:%d:0x%03x)\n",
		s->addr_cena, s->idx, offset);
#endif
	QBMAN_BUG_ON(offset & 63);
#ifdef RTE_ARCH_64
	return (s->addr_cena + offset);
#else
	return (s->addr_cinh + offset);
#endif
}

static inline void qbman_cena_write_complete(struct qbman_swp_sys *s,
					     uint32_t offset, void *cmd)
{
	const uint32_t *shadow = cmd;
	int loop;
#ifdef QBMAN_CENA_TRACE
	pr_info("qbman_cena_write_complete(%p:%d:0x%03x) %p\n",
		s->addr_cena, s->idx, offset, shadow);
	hexdump(cmd, 64);
#endif
#ifdef RTE_ARCH_64
	for (loop = 15; loop >= 1; loop--)
		__raw_writel(shadow[loop], s->addr_cena +
					 offset + loop * 4);
	lwsync();
		__raw_writel(shadow[0], s->addr_cena + offset);
#else
	for (loop = 15; loop >= 1; loop--)
		__raw_writel(shadow[loop], s->addr_cinh +
					 offset + loop * 4);
	lwsync();
	__raw_writel(shadow[0], s->addr_cinh + offset);
#endif
	dcbf(s->addr_cena + offset);
}

static inline void qbman_cena_write_complete_wo_shadow(struct qbman_swp_sys *s,
						       uint32_t offset)
{
#ifdef QBMAN_CENA_TRACE
	pr_info("qbman_cena_write_complete(%p:%d:0x%03x)\n",
		s->addr_cena, s->idx, offset);
#endif
	dcbf(s->addr_cena + offset);
}

static inline uint32_t qbman_cena_read_reg(struct qbman_swp_sys *s,
					   uint32_t offset)
{
	return __raw_readl(s->addr_cena + offset);
}

static inline void *qbman_cena_read(struct qbman_swp_sys *s, uint32_t offset)
{
	uint32_t *shadow = (uint32_t *)(s->cena + offset);
	unsigned int loop;
#ifdef QBMAN_CENA_TRACE
	pr_info("qbman_cena_read(%p:%d:0x%03x) %p\n",
		s->addr_cena, s->idx, offset, shadow);
#endif

#ifdef RTE_ARCH_64
	for (loop = 0; loop < 16; loop++)
		shadow[loop] = __raw_readl(s->addr_cena + offset
					+ loop * 4);
#else
	for (loop = 0; loop < 16; loop++)
		shadow[loop] = __raw_readl(s->addr_cinh + offset
					+ loop * 4);
#endif
#ifdef QBMAN_CENA_TRACE
	hexdump(shadow, 64);
#endif
	return shadow;
}

static inline void *qbman_cena_read_wo_shadow(struct qbman_swp_sys *s,
					      uint32_t offset)
{
#ifdef QBMAN_CENA_TRACE
	pr_info("qbman_cena_read(%p:%d:0x%03x)\n",
		s->addr_cena, s->idx, offset);
#endif
	return s->addr_cena + offset;
}

static inline void qbman_cena_invalidate(struct qbman_swp_sys *s,
					 uint32_t offset)
{
	dccivac(s->addr_cena + offset);
}

static inline void qbman_cena_invalidate_prefetch(struct qbman_swp_sys *s,
						  uint32_t offset)
{
	dccivac(s->addr_cena + offset);
	prefetch_for_load(s->addr_cena + offset);
}

static inline void qbman_cena_prefetch(struct qbman_swp_sys *s,
				       uint32_t offset)
{
	prefetch_for_load(s->addr_cena + offset);
}

	/******************/
	/* Portal support */
	/******************/

/* The SWP_CFG portal register is special, in that it is used by the
 * platform-specific code rather than the platform-independent code in
 * qbman_portal.c. So use of it is declared locally here.
 */
#define QBMAN_CINH_SWP_CFG   0xd00

#define SWP_CFG_DQRR_MF_SHIFT 20
#define SWP_CFG_EST_SHIFT     16
#define SWP_CFG_CPBS_SHIFT    15
#define SWP_CFG_WN_SHIFT      14
#define SWP_CFG_RPM_SHIFT     12
#define SWP_CFG_DCM_SHIFT     10
#define SWP_CFG_EPM_SHIFT     8
#define SWP_CFG_VPM_SHIFT     7
#define SWP_CFG_CPM_SHIFT     6
#define SWP_CFG_SD_SHIFT      5
#define SWP_CFG_SP_SHIFT      4
#define SWP_CFG_SE_SHIFT      3
#define SWP_CFG_DP_SHIFT      2
#define SWP_CFG_DE_SHIFT      1
#define SWP_CFG_EP_SHIFT      0

static inline uint32_t qbman_set_swp_cfg(uint8_t max_fill, uint8_t wn,
					 uint8_t est, uint8_t rpm, uint8_t dcm,
					uint8_t epm, int sd, int sp, int se,
					int dp, int de, int ep)
{
	uint32_t reg;

	reg = (max_fill << SWP_CFG_DQRR_MF_SHIFT |
		est << SWP_CFG_EST_SHIFT |
		wn << SWP_CFG_WN_SHIFT |
		rpm << SWP_CFG_RPM_SHIFT |
		dcm << SWP_CFG_DCM_SHIFT |
		epm << SWP_CFG_EPM_SHIFT |
		sd << SWP_CFG_SD_SHIFT |
		sp << SWP_CFG_SP_SHIFT |
		se << SWP_CFG_SE_SHIFT |
		dp << SWP_CFG_DP_SHIFT |
		de << SWP_CFG_DE_SHIFT |
		ep << SWP_CFG_EP_SHIFT);

	return reg;
}

#define QMAN_RT_MODE	0x00000100

#define QMAN_REV_4000	0x04000000
#define QMAN_REV_4100	0x04010000
#define QMAN_REV_4101	0x04010001
#define QMAN_REV_5000	0x05000000
#define QMAN_REV_MASK	0xffff0000

static inline int qbman_swp_sys_init(struct qbman_swp_sys *s,
				     const struct qbman_swp_desc *d,
				     uint8_t dqrr_size)
{
	uint32_t reg;
	int i;
	int cena_region_size = 4*1024;

	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
			&& (d->cena_access_mode == qman_cena_fastest_access))
		cena_region_size = 64*1024;
#ifdef RTE_ARCH_64
	uint8_t wn = CENA_WRITE_ENABLE;
#else
	uint8_t wn = CINH_WRITE_ENABLE;
#endif

	s->addr_cena = d->cena_bar;
	s->addr_cinh = d->cinh_bar;
	s->idx = (uint32_t)d->idx;
	s->cena = malloc(cena_region_size);

	if (!s->cena) {
		pr_err("Could not allocate page for cena shadow\n");
		return -1;
	}
	s->eqcr_mode = d->eqcr_mode;
	QBMAN_BUG_ON(d->idx < 0);
#ifdef QBMAN_CHECKING
	/* We should never be asked to initialise for a portal that isn't in
	 * the power-on state. (Ie. don't forget to reset portals when they are
	 * decommissioned!)
	 */
	reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG);
	QBMAN_BUG_ON(reg);
#endif
	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
			&& (d->cena_access_mode == qman_cena_fastest_access))
		memset(s->addr_cena, 0, cena_region_size);
	else {
		/* Invalidate the portal memory.
		 * This ensures no stale cache lines
		 */
		for (i = 0; i < cena_region_size; i += 64)
			dccivac(s->addr_cena + i);
	}

	if (s->eqcr_mode == qman_eqcr_vb_array) {
		reg = qbman_set_swp_cfg(dqrr_size, wn,
					0, 3, 2, 3, 1, 1, 1, 1, 1, 1);
	} else {
		if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000 &&
			    (d->cena_access_mode == qman_cena_fastest_access))
			reg = qbman_set_swp_cfg(dqrr_size, wn,
						1, 3, 2, 0, 1, 1, 1, 1, 1, 1);
		else
			reg = qbman_set_swp_cfg(dqrr_size, wn,
						1, 3, 2, 2, 1, 1, 1, 1, 1, 1);
	}

	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
			&& (d->cena_access_mode == qman_cena_fastest_access))
		reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */
		       1 << SWP_CFG_VPM_SHIFT |  /* VDQCR read triggered mode */
		       1 << SWP_CFG_CPM_SHIFT;   /* CR read triggered mode */

	qbman_cinh_write(s, QBMAN_CINH_SWP_CFG, reg);
	reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG);
	if (!reg) {
		pr_err("The portal %d is not enabled!\n", s->idx);
		free(s->cena);
		return -1;
	}

	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
			&& (d->cena_access_mode == qman_cena_fastest_access)) {
		qbman_cinh_write(s, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE);
		qbman_cinh_write(s, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE);
	}

	return 0;
}

static inline void qbman_swp_sys_finish(struct qbman_swp_sys *s)
{
	free(s->cena);
}

#endif /* _QBMAN_SYS_H_ */
Commit	Line	Data
11fdf7f2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause
	2	*
	3	* Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
	4	*
	5	*/
	6	/* qbman_sys_decl.h and qbman_sys.h are the two platform-specific files in the
	7	* driver. They are only included via qbman_private.h, which is itself a
	8	* platform-independent file and is included by all the other driver source.
	9	*
	10	* qbman_sys_decl.h is included prior to all other declarations and logic, and
	11	* it exists to provide compatibility with any linux interfaces our
	12	* single-source driver code is dependent on (eg. kmalloc). Ie. this file
	13	* provides linux compatibility.
	14	*
	15	* This qbman_sys.h header, on the other hand, is included after any common
	16	* and platform-neutral declarations and logic in qbman_private.h, and exists to
	17	* implement any platform-specific logic of the qbman driver itself. Ie. it is
	18	* not to provide linux compatibility.
	19	*/
	20
9f95a23c TL	21	#ifndef _QBMAN_SYS_H_
	22	#define _QBMAN_SYS_H_
	23
11fdf7f2 TL	24	#include "qbman_sys_decl.h"
	25
	26	#define CENA_WRITE_ENABLE 0
	27	#define CINH_WRITE_ENABLE 1
	28
9f95a23c TL	29	/* CINH register offsets */
	30	#define QBMAN_CINH_SWP_EQCR_PI 0x800
	31	#define QBMAN_CINH_SWP_EQCR_CI 0x840
	32	#define QBMAN_CINH_SWP_EQAR 0x8c0
	33	#define QBMAN_CINH_SWP_CR_RT 0x900
	34	#define QBMAN_CINH_SWP_VDQCR_RT 0x940
	35	#define QBMAN_CINH_SWP_EQCR_AM_RT 0x980
	36	#define QBMAN_CINH_SWP_RCR_AM_RT 0x9c0
	37	#define QBMAN_CINH_SWP_DQPI 0xa00
	38	#define QBMAN_CINH_SWP_DQRR_ITR 0xa80
	39	#define QBMAN_CINH_SWP_DCAP 0xac0
	40	#define QBMAN_CINH_SWP_SDQCR 0xb00
	41	#define QBMAN_CINH_SWP_EQCR_AM_RT2 0xb40
	42	#define QBMAN_CINH_SWP_RCR_PI 0xc00
	43	#define QBMAN_CINH_SWP_RAR 0xcc0
	44	#define QBMAN_CINH_SWP_ISR 0xe00
	45	#define QBMAN_CINH_SWP_IER 0xe40
	46	#define QBMAN_CINH_SWP_ISDR 0xe80
	47	#define QBMAN_CINH_SWP_IIR 0xec0
	48	#define QBMAN_CINH_SWP_ITPR 0xf40
	49
	50	/* CENA register offsets */
	51	#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6))
	52	#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6))
	53	#define QBMAN_CENA_SWP_RCR(n) (0x400 + ((uint32_t)(n) << 6))
	54	#define QBMAN_CENA_SWP_CR 0x600
	55	#define QBMAN_CENA_SWP_RR(vb) (0x700 + ((uint32_t)(vb) >> 1))
	56	#define QBMAN_CENA_SWP_VDQCR 0x780
	57	#define QBMAN_CENA_SWP_EQCR_CI 0x840
	58	#define QBMAN_CENA_SWP_EQCR_CI_MEMBACK 0x1840
	59
	60	/* CENA register offsets in memory-backed mode */
	61	#define QBMAN_CENA_SWP_DQRR_MEM(n) (0x800 + ((uint32_t)(n) << 6))
	62	#define QBMAN_CENA_SWP_RCR_MEM(n) (0x1400 + ((uint32_t)(n) << 6))
	63	#define QBMAN_CENA_SWP_CR_MEM 0x1600
	64	#define QBMAN_CENA_SWP_RR_MEM 0x1680
	65	#define QBMAN_CENA_SWP_VDQCR_MEM 0x1780
	66
11fdf7f2 TL	67	/* Debugging assists */
	68	static inline void __hexdump(unsigned long start, unsigned long end,
	69	unsigned long p, size_t sz, const unsigned char *c)
	70	{
	71	while (start < end) {
	72	unsigned int pos = 0;
	73	char buf[64];
	74	int nl = 0;
	75
	76	pos += sprintf(buf + pos, "%08lx: ", start);
	77	do {
	78	if ((start < p) \|\| (start >= (p + sz)))
	79	pos += sprintf(buf + pos, "..");
	80	else
	81	pos += sprintf(buf + pos, "%02x", *(c++));
	82	if (!(++start & 15)) {
	83	buf[pos++] = '\n';
	84	nl = 1;
	85	} else {
	86	nl = 0;
	87	if (!(start & 1))
	88	buf[pos++] = ' ';
	89	if (!(start & 3))
	90	buf[pos++] = ' ';
	91	}
	92	} while (start & 15);
	93	if (!nl)
	94	buf[pos++] = '\n';
	95	buf[pos] = '\0';
	96	pr_info("%s", buf);
	97	}
	98	}
	99
	100	static inline void hexdump(const void *ptr, size_t sz)
	101	{
	102	unsigned long p = (unsigned long)ptr;
	103	unsigned long start = p & ~15;
	104	unsigned long end = (p + sz + 15) & ~15;
	105	const unsigned char *c = ptr;
	106
	107	__hexdump(start, end, p, sz, c);
	108	}
	109
	110	/* Currently, the CENA support code expects each 32-bit word to be written in
	111	* host order, and these are converted to hardware (little-endian) order on
	112	* command submission. However, 64-bit quantities are must be written (and read)
	113	* as two 32-bit words with the least-significant word first, irrespective of
	114	* host endianness.
	115	*/
	116	static inline void u64_to_le32_copy(void d, const uint64_t s,
	117	unsigned int cnt)
	118	{
	119	uint32_t *dd = d;
	120	const uint32_t ss = (const uint32_t )s;
	121
	122	while (cnt--) {
	123	/* TBD: the toolchain was choking on the use of 64-bit types up
	124	* until recently so this works entirely with 32-bit variables.
	125	* When 64-bit types become usable again, investigate better
	126	* ways of doing this.
	127	*/
	128	#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	129	*(dd++) = ss[1];
	130	*(dd++) = ss[0];
131	ss += 2;
132	#else
133	(dd++) = (ss++);
134	(dd++) = (ss++);
135	#endif
136	}
137	}
138
139	static inline void u64_from_le32_copy(uint64_t d, const void s,
140	unsigned int cnt)
141	{
142	const uint32_t *ss = s;
143	uint32_t dd = (uint32_t )d;
144
145	while (cnt--) {
146	#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
147	dd[1] = *(ss++);
148	dd[0] = *(ss++);
149	dd += 2;
150	#else
151	(dd++) = (ss++);
152	(dd++) = (ss++);
153	#endif
154	}
155	}
156
157	/******************/
158	/* Portal access */
159	/******************/
160	struct qbman_swp_sys {
161	/* On GPP, the sys support for qbman_swp is here. The CENA region isi
162	* not an mmap() of the real portal registers, but an allocated
163	* place-holder, because the actual writes/reads to/from the portal are
164	* marshalled from these allocated areas using QBMan's "MC access
165	* registers". CINH accesses are atomic so there's no need for a
166	* place-holder.
167	*/
168	uint8_t *cena;
9f95a23c TL	169	uint8_t *addr_cena;
9f95a23c TL	170	uint8_t *addr_cinh;
11fdf7f2 TL	171	uint32_t idx;
	172	enum qbman_eqcr_mode eqcr_mode;
	173	};
	174
	175	/* P_OFFSET is (ACCESS_CMD,0,12) - offset within the portal
	176	* C is (ACCESS_CMD,12,1) - is inhibited? (0==CENA, 1==CINH)
	177	* SWP_IDX is (ACCESS_CMD,16,10) - Software portal index
	178	* P is (ACCESS_CMD,28,1) - (0==special portal, 1==any portal)
	179	* T is (ACCESS_CMD,29,1) - Command type (0==READ, 1==WRITE)
	180	* E is (ACCESS_CMD,31,1) - Command execute (1 to issue, poll for 0==complete)
	181	*/
	182
	183	static inline void qbman_cinh_write(struct qbman_swp_sys *s, uint32_t offset,
	184	uint32_t val)
	185	{
	186	__raw_writel(val, s->addr_cinh + offset);
	187	#ifdef QBMAN_CINH_TRACE
	188	pr_info("qbman_cinh_write(%p:%d:0x%03x) 0x%08x\n",
	189	s->addr_cinh, s->idx, offset, val);
	190	#endif
	191	}
	192
	193	static inline uint32_t qbman_cinh_read(struct qbman_swp_sys *s, uint32_t offset)
	194	{
	195	uint32_t reg = __raw_readl(s->addr_cinh + offset);
	196	#ifdef QBMAN_CINH_TRACE
	197	pr_info("qbman_cinh_read(%p:%d:0x%03x) 0x%08x\n",
	198	s->addr_cinh, s->idx, offset, reg);
	199	#endif
	200	return reg;
	201	}
	202
	203	static inline void qbman_cena_write_start(struct qbman_swp_sys s,
	204	uint32_t offset)
	205	{
	206	void *shadow = s->cena + offset;
	207
	208	#ifdef QBMAN_CENA_TRACE
	209	pr_info("qbman_cena_write_start(%p:%d:0x%03x) %p\n",
	210	s->addr_cena, s->idx, offset, shadow);
	211	#endif
	212	QBMAN_BUG_ON(offset & 63);
	213	dcbz(shadow);
	214	return shadow;
	215	}
	216
	217	static inline void qbman_cena_write_start_wo_shadow(struct qbman_swp_sys s,
	218	uint32_t offset)
	219	{
	220	#ifdef QBMAN_CENA_TRACE
	221	pr_info("qbman_cena_write_start(%p:%d:0x%03x)\n",
	222	s->addr_cena, s->idx, offset);
	223	#endif
	224	QBMAN_BUG_ON(offset & 63);
	225	#ifdef RTE_ARCH_64
	226	return (s->addr_cena + offset);
	227	#else
	228	return (s->addr_cinh + offset);
	229	#endif
	230	}
	231
	232	static inline void qbman_cena_write_complete(struct qbman_swp_sys *s,
	233	uint32_t offset, void *cmd)
	234	{
235	const uint32_t *shadow = cmd;
236	int loop;
237	#ifdef QBMAN_CENA_TRACE
238	pr_info("qbman_cena_write_complete(%p:%d:0x%03x) %p\n",
239	s->addr_cena, s->idx, offset, shadow);
240	hexdump(cmd, 64);
241	#endif
242	#ifdef RTE_ARCH_64
243	for (loop = 15; loop >= 1; loop--)
244	__raw_writel(shadow[loop], s->addr_cena +
245	offset + loop * 4);
246	lwsync();
247	__raw_writel(shadow[0], s->addr_cena + offset);
248	#else
249	for (loop = 15; loop >= 1; loop--)
250	__raw_writel(shadow[loop], s->addr_cinh +
251	offset + loop * 4);
252	lwsync();
253	__raw_writel(shadow[0], s->addr_cinh + offset);
254	#endif
255	dcbf(s->addr_cena + offset);
256	}
257
258	static inline void qbman_cena_write_complete_wo_shadow(struct qbman_swp_sys *s,
259	uint32_t offset)
260	{
261	#ifdef QBMAN_CENA_TRACE
262	pr_info("qbman_cena_write_complete(%p:%d:0x%03x)\n",
263	s->addr_cena, s->idx, offset);
264	#endif
265	dcbf(s->addr_cena + offset);
266	}
267
268	static inline uint32_t qbman_cena_read_reg(struct qbman_swp_sys *s,
269	uint32_t offset)
270	{
271	return __raw_readl(s->addr_cena + offset);
272	}
273
274	static inline void qbman_cena_read(struct qbman_swp_sys s, uint32_t offset)
275	{
276	uint32_t shadow = (uint32_t )(s->cena + offset);
277	unsigned int loop;
278	#ifdef QBMAN_CENA_TRACE
279	pr_info("qbman_cena_read(%p:%d:0x%03x) %p\n",
280	s->addr_cena, s->idx, offset, shadow);
281	#endif
282
283	#ifdef RTE_ARCH_64
284	for (loop = 0; loop < 16; loop++)
285	shadow[loop] = __raw_readl(s->addr_cena + offset
286	+ loop * 4);
287	#else
288	for (loop = 0; loop < 16; loop++)
289	shadow[loop] = __raw_readl(s->addr_cinh + offset
290	+ loop * 4);
291	#endif
292	#ifdef QBMAN_CENA_TRACE
293	hexdump(shadow, 64);
294	#endif
295	return shadow;
296	}
297
298	static inline void qbman_cena_read_wo_shadow(struct qbman_swp_sys s,
299	uint32_t offset)
300	{
301	#ifdef QBMAN_CENA_TRACE
302	pr_info("qbman_cena_read(%p:%d:0x%03x)\n",
303	s->addr_cena, s->idx, offset);
304	#endif
305	return s->addr_cena + offset;
306	}
307
308	static inline void qbman_cena_invalidate(struct qbman_swp_sys *s,
309	uint32_t offset)
310	{
311	dccivac(s->addr_cena + offset);
312	}
313
314	static inline void qbman_cena_invalidate_prefetch(struct qbman_swp_sys *s,
315	uint32_t offset)
316	{
317	dccivac(s->addr_cena + offset);
318	prefetch_for_load(s->addr_cena + offset);
319	}
320
321	static inline void qbman_cena_prefetch(struct qbman_swp_sys *s,
322	uint32_t offset)
323	{
324	prefetch_for_load(s->addr_cena + offset);
325	}
326
327	/******************/
328	/* Portal support */
329	/******************/
330
331	/* The SWP_CFG portal register is special, in that it is used by the
332	* platform-specific code rather than the platform-independent code in
333	* qbman_portal.c. So use of it is declared locally here.
334	*/
335	#define QBMAN_CINH_SWP_CFG 0xd00
9f95a23c	336
11fdf7f2 TL	337	#define SWP_CFG_DQRR_MF_SHIFT 20
11fdf7f2 TL	338	#define SWP_CFG_EST_SHIFT 16
9f95a23c	339	#define SWP_CFG_CPBS_SHIFT 15
11fdf7f2 TL	340	#define SWP_CFG_WN_SHIFT 14
	341	#define SWP_CFG_RPM_SHIFT 12
	342	#define SWP_CFG_DCM_SHIFT 10
	343	#define SWP_CFG_EPM_SHIFT 8
9f95a23c TL	344	#define SWP_CFG_VPM_SHIFT 7
9f95a23c TL	345	#define SWP_CFG_CPM_SHIFT 6
11fdf7f2 TL	346	#define SWP_CFG_SD_SHIFT 5
	347	#define SWP_CFG_SP_SHIFT 4
	348	#define SWP_CFG_SE_SHIFT 3
	349	#define SWP_CFG_DP_SHIFT 2
	350	#define SWP_CFG_DE_SHIFT 1
	351	#define SWP_CFG_EP_SHIFT 0
	352
	353	static inline uint32_t qbman_set_swp_cfg(uint8_t max_fill, uint8_t wn,
	354	uint8_t est, uint8_t rpm, uint8_t dcm,
	355	uint8_t epm, int sd, int sp, int se,
	356	int dp, int de, int ep)
	357	{
	358	uint32_t reg;
	359
	360	reg = (max_fill << SWP_CFG_DQRR_MF_SHIFT \|
	361	est << SWP_CFG_EST_SHIFT \|
	362	wn << SWP_CFG_WN_SHIFT \|
	363	rpm << SWP_CFG_RPM_SHIFT \|
	364	dcm << SWP_CFG_DCM_SHIFT \|
	365	epm << SWP_CFG_EPM_SHIFT \|
	366	sd << SWP_CFG_SD_SHIFT \|
	367	sp << SWP_CFG_SP_SHIFT \|
	368	se << SWP_CFG_SE_SHIFT \|
	369	dp << SWP_CFG_DP_SHIFT \|
	370	de << SWP_CFG_DE_SHIFT \|
	371	ep << SWP_CFG_EP_SHIFT);
	372
	373	return reg;
	374	}
	375
9f95a23c TL	376	#define QMAN_RT_MODE 0x00000100
	377
	378	#define QMAN_REV_4000 0x04000000
	379	#define QMAN_REV_4100 0x04010000
	380	#define QMAN_REV_4101 0x04010001
	381	#define QMAN_REV_5000 0x05000000
	382	#define QMAN_REV_MASK 0xffff0000
	383
11fdf7f2 TL	384	static inline int qbman_swp_sys_init(struct qbman_swp_sys *s,
	385	const struct qbman_swp_desc *d,
	386	uint8_t dqrr_size)
	387	{
	388	uint32_t reg;
9f95a23c TL	389	int i;
	390	int cena_region_size = 4*1024;
	391
	392	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
	393	&& (d->cena_access_mode == qman_cena_fastest_access))
	394	cena_region_size = 64*1024;
11fdf7f2 TL	395	#ifdef RTE_ARCH_64
	396	uint8_t wn = CENA_WRITE_ENABLE;
	397	#else
	398	uint8_t wn = CINH_WRITE_ENABLE;
	399	#endif
	400
	401	s->addr_cena = d->cena_bar;
	402	s->addr_cinh = d->cinh_bar;
	403	s->idx = (uint32_t)d->idx;
9f95a23c TL	404	s->cena = malloc(cena_region_size);
9f95a23c TL	405
11fdf7f2 TL	406	if (!s->cena) {
	407	pr_err("Could not allocate page for cena shadow\n");
	408	return -1;
	409	}
	410	s->eqcr_mode = d->eqcr_mode;
	411	QBMAN_BUG_ON(d->idx < 0);
	412	#ifdef QBMAN_CHECKING
	413	/* We should never be asked to initialise for a portal that isn't in
	414	* the power-on state. (Ie. don't forget to reset portals when they are
	415	* decommissioned!)
	416	*/
	417	reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG);
	418	QBMAN_BUG_ON(reg);
	419	#endif
9f95a23c TL	420	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
	421	&& (d->cena_access_mode == qman_cena_fastest_access))
	422	memset(s->addr_cena, 0, cena_region_size);
	423	else {
	424	/* Invalidate the portal memory.
	425	* This ensures no stale cache lines
	426	*/
	427	for (i = 0; i < cena_region_size; i += 64)
	428	dccivac(s->addr_cena + i);
	429	}
	430
	431	if (s->eqcr_mode == qman_eqcr_vb_array) {
	432	reg = qbman_set_swp_cfg(dqrr_size, wn,
	433	0, 3, 2, 3, 1, 1, 1, 1, 1, 1);
	434	} else {
	435	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000 &&
	436	(d->cena_access_mode == qman_cena_fastest_access))
	437	reg = qbman_set_swp_cfg(dqrr_size, wn,
	438	1, 3, 2, 0, 1, 1, 1, 1, 1, 1);
	439	else
	440	reg = qbman_set_swp_cfg(dqrr_size, wn,
	441	1, 3, 2, 2, 1, 1, 1, 1, 1, 1);
	442	}
	443
	444	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
	445	&& (d->cena_access_mode == qman_cena_fastest_access))
	446	reg \|= 1 << SWP_CFG_CPBS_SHIFT \| /* memory-backed mode */
	447	1 << SWP_CFG_VPM_SHIFT \| /* VDQCR read triggered mode */
	448	1 << SWP_CFG_CPM_SHIFT; /* CR read triggered mode */
	449
11fdf7f2 TL	450	qbman_cinh_write(s, QBMAN_CINH_SWP_CFG, reg);
	451	reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG);
	452	if (!reg) {
	453	pr_err("The portal %d is not enabled!\n", s->idx);
	454	free(s->cena);
	455	return -1;
	456	}
9f95a23c TL	457
	458	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
	459	&& (d->cena_access_mode == qman_cena_fastest_access)) {
	460	qbman_cinh_write(s, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE);
	461	qbman_cinh_write(s, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE);
	462	}
	463
11fdf7f2 TL	464	return 0;
	465	}
	466
	467	static inline void qbman_swp_sys_finish(struct qbman_swp_sys *s)
	468	{
	469	free(s->cena);
	470	}
9f95a23c TL	471
9f95a23c TL	472	#endif /* _QBMAN_SYS_H_ */