[ceph.git] / ceph / src / spdk / dpdk / drivers / event / octeontx / timvf_evdev.h

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2017 Cavium, Inc
 */

#ifndef __TIMVF_EVDEV_H__
#define __TIMVF_EVDEV_H__

#include <rte_common.h>
#include <rte_cycles.h>
#include <rte_debug.h>
#include <rte_eal.h>
#include <rte_eventdev.h>
#include <rte_event_timer_adapter.h>
#include <rte_event_timer_adapter_pmd.h>
#include <rte_io.h>
#include <rte_lcore.h>
#include <rte_log.h>
#include <rte_malloc.h>
#include <rte_mbuf_pool_ops.h>
#include <rte_mempool.h>
#include <rte_memzone.h>
#include <rte_pci.h>
#include <rte_prefetch.h>
#include <rte_reciprocal.h>

#include <octeontx_mbox.h>
#include <octeontx_fpavf.h>

#define timvf_log(level, fmt, args...) \
	rte_log(RTE_LOG_ ## level, otx_logtype_timvf, \
			"[%s] %s() " fmt "\n", \
			RTE_STR(event_timer_octeontx), __func__, ## args)

#define timvf_log_info(fmt, ...) timvf_log(INFO, fmt, ##__VA_ARGS__)
#define timvf_log_dbg(fmt, ...) timvf_log(DEBUG, fmt, ##__VA_ARGS__)
#define timvf_log_err(fmt, ...) timvf_log(ERR, fmt, ##__VA_ARGS__)
#define timvf_func_trace timvf_log_dbg

#define TIM_COPROC				(8)
#define TIM_GET_DEV_INFO			(1)
#define TIM_GET_RING_INFO			(2)
#define TIM_SET_RING_INFO			(3)
#define TIM_RING_START_CYC_GET			(4)

#define TIM_MAX_RINGS				(64)
#define TIM_DEV_PER_NODE			(1)
#define TIM_VF_PER_DEV				(64)
#define TIM_RING_PER_DEV			(TIM_VF_PER_DEV)
#define TIM_RING_NODE_SHIFT			(6)
#define TIM_RING_MASK				((TIM_RING_PER_DEV) - 1)
#define TIM_RING_INVALID			(-1)

#define TIM_MIN_INTERVAL			(1E3)
#define TIM_MAX_INTERVAL			((1ull << 32) - 1)
#define TIM_MAX_BUCKETS				(1ull << 20)
#define TIM_CHUNK_SIZE				(4096)
#define TIM_MAX_CHUNKS_PER_BUCKET		(1ull << 32)

#define TIMVF_MAX_BURST				(8)

/* TIM VF Control/Status registers (CSRs): */
/* VF_BAR0: */
#define TIM_VF_NRSPERR_INT			(0x0)
#define TIM_VF_NRSPERR_INT_W1S			(0x8)
#define TIM_VF_NRSPERR_ENA_W1C			(0x10)
#define TIM_VF_NRSPERR_ENA_W1S			(0x18)
#define TIM_VRING_FR_RN_CYCLES			(0x20)
#define TIM_VRING_FR_RN_GPIOS			(0x28)
#define TIM_VRING_FR_RN_GTI			(0x30)
#define TIM_VRING_FR_RN_PTP			(0x38)
#define TIM_VRING_CTL0				(0x40)
#define TIM_VRING_CTL1				(0x50)
#define TIM_VRING_CTL2				(0x60)
#define TIM_VRING_BASE				(0x100)
#define TIM_VRING_AURA				(0x108)
#define TIM_VRING_REL				(0x110)

#define TIM_CTL1_W0_S_BUCKET			20
#define TIM_CTL1_W0_M_BUCKET			((1ull << (40 - 20)) - 1)

#define TIM_BUCKET_W1_S_NUM_ENTRIES		(0) /*Shift*/
#define TIM_BUCKET_W1_M_NUM_ENTRIES		((1ull << (32 - 0)) - 1)
#define TIM_BUCKET_W1_S_SBT			(32)
#define TIM_BUCKET_W1_M_SBT			((1ull << (33 - 32)) - 1)
#define TIM_BUCKET_W1_S_HBT			(33)
#define TIM_BUCKET_W1_M_HBT			((1ull << (34 - 33)) - 1)
#define TIM_BUCKET_W1_S_BSK			(34)
#define TIM_BUCKET_W1_M_BSK			((1ull << (35 - 34)) - 1)
#define TIM_BUCKET_W1_S_LOCK			(40)
#define TIM_BUCKET_W1_M_LOCK			((1ull << (48 - 40)) - 1)
#define TIM_BUCKET_W1_S_CHUNK_REMAINDER		(48)
#define TIM_BUCKET_W1_M_CHUNK_REMAINDER		((1ull << (64 - 48)) - 1)

#define TIM_BUCKET_SEMA	\
	(TIM_BUCKET_CHUNK_REMAIN)

#define TIM_BUCKET_CHUNK_REMAIN \
	(TIM_BUCKET_W1_M_CHUNK_REMAINDER << TIM_BUCKET_W1_S_CHUNK_REMAINDER)

#define TIM_BUCKET_LOCK \
	(TIM_BUCKET_W1_M_LOCK << TIM_BUCKET_W1_S_LOCK)

#define TIM_BUCKET_SEMA_WLOCK \
	(TIM_BUCKET_CHUNK_REMAIN | (1ull << TIM_BUCKET_W1_S_LOCK))

#define NSEC_PER_SEC 1E9
#define NSEC2CLK(__ns, __freq) (((__ns) * (__freq)) / NSEC_PER_SEC)
#define CLK2NSEC(__clk, __freq) (((__clk) * NSEC_PER_SEC) / (__freq))

#define timvf_read64 rte_read64_relaxed
#define timvf_write64 rte_write64_relaxed

#define TIMVF_ENABLE_STATS_ARG               ("timvf_stats")

extern int otx_logtype_timvf;
static const uint16_t nb_chunk_slots = (TIM_CHUNK_SIZE / 16) - 1;

enum timvf_clk_src {
	TIM_CLK_SRC_SCLK = RTE_EVENT_TIMER_ADAPTER_CPU_CLK,
	TIM_CLK_SRC_GPIO = RTE_EVENT_TIMER_ADAPTER_EXT_CLK0,
	TIM_CLK_SRC_GTI = RTE_EVENT_TIMER_ADAPTER_EXT_CLK1,
	TIM_CLK_SRC_PTP = RTE_EVENT_TIMER_ADAPTER_EXT_CLK2,
};

/* TIM_MEM_BUCKET */
struct tim_mem_bucket {
	uint64_t first_chunk;
	union {
		uint64_t w1;
		struct {
			uint32_t nb_entry;
			uint8_t sbt:1;
			uint8_t hbt:1;
			uint8_t bsk:1;
			uint8_t rsvd:5;
			uint8_t lock;
			int16_t chunk_remainder;
		};
	};
	uint64_t current_chunk;
	uint64_t pad;
} __rte_packed __rte_aligned(8);

struct tim_mem_entry {
	uint64_t w0;
	uint64_t wqe;
} __rte_packed;

struct timvf_ctrl_reg {
	uint64_t rctrl0;
	uint64_t rctrl1;
	uint64_t rctrl2;
	uint8_t use_pmu;
} __rte_packed;

struct timvf_ring;

typedef uint32_t (*bkt_id)(const uint32_t bkt_tcks, const uint32_t nb_bkts);
typedef struct tim_mem_entry * (*refill_chunk)(
		struct tim_mem_bucket * const bkt,
		struct timvf_ring * const timr);

struct timvf_ring {
	bkt_id get_target_bkt;
	refill_chunk refill_chunk;
	struct rte_reciprocal_u64 fast_div;
	uint64_t ring_start_cyc;
	uint32_t nb_bkts;
	struct tim_mem_bucket *bkt;
	void *chunk_pool;
	uint64_t tck_int;
	volatile uint64_t tim_arm_cnt;
	uint64_t tck_nsec;
	void  *vbar0;
	void *bkt_pos;
	uint64_t max_tout;
	uint64_t nb_chunks;
	enum timvf_clk_src clk_src;
	uint16_t tim_ring_id;
} __rte_cache_aligned;

static __rte_always_inline uint32_t
bkt_mod(const uint32_t rel_bkt, const uint32_t nb_bkts)
{
	return rel_bkt % nb_bkts;
}

static __rte_always_inline uint32_t
bkt_and(uint32_t rel_bkt, uint32_t nb_bkts)
{
	return rel_bkt & (nb_bkts - 1);
}

uint8_t timvf_get_ring(void);
void timvf_release_ring(uint8_t vfid);
void *timvf_bar(uint8_t id, uint8_t bar);
int timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags,
		uint32_t *caps, const struct rte_event_timer_adapter_ops **ops,
		uint8_t enable_stats);
uint16_t timvf_timer_cancel_burst(const struct rte_event_timer_adapter *adptr,
		struct rte_event_timer **tim, const uint16_t nb_timers);
uint16_t timvf_timer_arm_burst_sp(const struct rte_event_timer_adapter *adptr,
		struct rte_event_timer **tim, const uint16_t nb_timers);
uint16_t timvf_timer_arm_burst_sp_stats(
		const struct rte_event_timer_adapter *adptr,
		struct rte_event_timer **tim, const uint16_t nb_timers);
uint16_t timvf_timer_arm_burst_mp(const struct rte_event_timer_adapter *adptr,
		struct rte_event_timer **tim, const uint16_t nb_timers);
uint16_t timvf_timer_arm_burst_mp_stats(
		const struct rte_event_timer_adapter *adptr,
		struct rte_event_timer **tim, const uint16_t nb_timers);
uint16_t timvf_timer_arm_tmo_brst(const struct rte_event_timer_adapter *adptr,
		struct rte_event_timer **tim, const uint64_t timeout_tick,
		const uint16_t nb_timers);
uint16_t timvf_timer_arm_tmo_brst_stats(
		const struct rte_event_timer_adapter *adptr,
		struct rte_event_timer **tim, const uint64_t timeout_tick,
		const uint16_t nb_timers);
void timvf_set_chunk_refill(struct timvf_ring * const timr, uint8_t use_fpa);

#endif /* __TIMVF_EVDEV_H__ */
Commit	Line	Data
11fdf7f2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause
	2	* Copyright(c) 2017 Cavium, Inc
	3	*/
	4
	5	#ifndef __TIMVF_EVDEV_H__
	6	#define __TIMVF_EVDEV_H__
	7
	8	#include <rte_common.h>
	9	#include <rte_cycles.h>
	10	#include <rte_debug.h>
	11	#include <rte_eal.h>
	12	#include <rte_eventdev.h>
	13	#include <rte_event_timer_adapter.h>
	14	#include <rte_event_timer_adapter_pmd.h>
	15	#include <rte_io.h>
	16	#include <rte_lcore.h>
	17	#include <rte_log.h>
	18	#include <rte_malloc.h>
	19	#include <rte_mbuf_pool_ops.h>
	20	#include <rte_mempool.h>
	21	#include <rte_memzone.h>
	22	#include <rte_pci.h>
	23	#include <rte_prefetch.h>
	24	#include <rte_reciprocal.h>
	25
	26	#include <octeontx_mbox.h>
	27	#include <octeontx_fpavf.h>
	28
	29	#define timvf_log(level, fmt, args...) \
	30	rte_log(RTE_LOG_ ## level, otx_logtype_timvf, \
	31	"[%s] %s() " fmt "\n", \
	32	RTE_STR(event_timer_octeontx), __func__, ## args)
	33
	34	#define timvf_log_info(fmt, ...) timvf_log(INFO, fmt, ##__VA_ARGS__)
	35	#define timvf_log_dbg(fmt, ...) timvf_log(DEBUG, fmt, ##__VA_ARGS__)
	36	#define timvf_log_err(fmt, ...) timvf_log(ERR, fmt, ##__VA_ARGS__)
	37	#define timvf_func_trace timvf_log_dbg
	38
	39	#define TIM_COPROC (8)
	40	#define TIM_GET_DEV_INFO (1)
	41	#define TIM_GET_RING_INFO (2)
	42	#define TIM_SET_RING_INFO (3)
	43	#define TIM_RING_START_CYC_GET (4)
	44
	45	#define TIM_MAX_RINGS (64)
	46	#define TIM_DEV_PER_NODE (1)
	47	#define TIM_VF_PER_DEV (64)
	48	#define TIM_RING_PER_DEV (TIM_VF_PER_DEV)
	49	#define TIM_RING_NODE_SHIFT (6)
	50	#define TIM_RING_MASK ((TIM_RING_PER_DEV) - 1)
	51	#define TIM_RING_INVALID (-1)
	52
	53	#define TIM_MIN_INTERVAL (1E3)
	54	#define TIM_MAX_INTERVAL ((1ull << 32) - 1)
	55	#define TIM_MAX_BUCKETS (1ull << 20)
	56	#define TIM_CHUNK_SIZE (4096)
	57	#define TIM_MAX_CHUNKS_PER_BUCKET (1ull << 32)
	58
	59	#define TIMVF_MAX_BURST (8)
	60
	61	/* TIM VF Control/Status registers (CSRs): */
	62	/* VF_BAR0: */
	63	#define TIM_VF_NRSPERR_INT (0x0)
	64	#define TIM_VF_NRSPERR_INT_W1S (0x8)
65	#define TIM_VF_NRSPERR_ENA_W1C (0x10)
66	#define TIM_VF_NRSPERR_ENA_W1S (0x18)
67	#define TIM_VRING_FR_RN_CYCLES (0x20)
68	#define TIM_VRING_FR_RN_GPIOS (0x28)
69	#define TIM_VRING_FR_RN_GTI (0x30)
70	#define TIM_VRING_FR_RN_PTP (0x38)
71	#define TIM_VRING_CTL0 (0x40)
72	#define TIM_VRING_CTL1 (0x50)
73	#define TIM_VRING_CTL2 (0x60)
74	#define TIM_VRING_BASE (0x100)
75	#define TIM_VRING_AURA (0x108)
76	#define TIM_VRING_REL (0x110)
77
78	#define TIM_CTL1_W0_S_BUCKET 20
79	#define TIM_CTL1_W0_M_BUCKET ((1ull << (40 - 20)) - 1)
80
81	#define TIM_BUCKET_W1_S_NUM_ENTRIES (0) /Shift/
82	#define TIM_BUCKET_W1_M_NUM_ENTRIES ((1ull << (32 - 0)) - 1)
83	#define TIM_BUCKET_W1_S_SBT (32)
84	#define TIM_BUCKET_W1_M_SBT ((1ull << (33 - 32)) - 1)
85	#define TIM_BUCKET_W1_S_HBT (33)
86	#define TIM_BUCKET_W1_M_HBT ((1ull << (34 - 33)) - 1)
87	#define TIM_BUCKET_W1_S_BSK (34)
88	#define TIM_BUCKET_W1_M_BSK ((1ull << (35 - 34)) - 1)
89	#define TIM_BUCKET_W1_S_LOCK (40)
90	#define TIM_BUCKET_W1_M_LOCK ((1ull << (48 - 40)) - 1)
91	#define TIM_BUCKET_W1_S_CHUNK_REMAINDER (48)
92	#define TIM_BUCKET_W1_M_CHUNK_REMAINDER ((1ull << (64 - 48)) - 1)
93
94	#define TIM_BUCKET_SEMA \
95	(TIM_BUCKET_CHUNK_REMAIN)
96
97	#define TIM_BUCKET_CHUNK_REMAIN \
98	(TIM_BUCKET_W1_M_CHUNK_REMAINDER << TIM_BUCKET_W1_S_CHUNK_REMAINDER)
99
100	#define TIM_BUCKET_LOCK \
101	(TIM_BUCKET_W1_M_LOCK << TIM_BUCKET_W1_S_LOCK)
102
103	#define TIM_BUCKET_SEMA_WLOCK \
104	(TIM_BUCKET_CHUNK_REMAIN \| (1ull << TIM_BUCKET_W1_S_LOCK))
105
106	#define NSEC_PER_SEC 1E9
107	#define NSEC2CLK(__ns, __freq) (((__ns) * (__freq)) / NSEC_PER_SEC)
108	#define CLK2NSEC(__clk, __freq) (((__clk) * NSEC_PER_SEC) / (__freq))
109
110	#define timvf_read64 rte_read64_relaxed
111	#define timvf_write64 rte_write64_relaxed
112
113	#define TIMVF_ENABLE_STATS_ARG ("timvf_stats")
114
115	extern int otx_logtype_timvf;
116	static const uint16_t nb_chunk_slots = (TIM_CHUNK_SIZE / 16) - 1;
117
11fdf7f2 TL	118	enum timvf_clk_src {
	119	TIM_CLK_SRC_SCLK = RTE_EVENT_TIMER_ADAPTER_CPU_CLK,
	120	TIM_CLK_SRC_GPIO = RTE_EVENT_TIMER_ADAPTER_EXT_CLK0,
	121	TIM_CLK_SRC_GTI = RTE_EVENT_TIMER_ADAPTER_EXT_CLK1,
	122	TIM_CLK_SRC_PTP = RTE_EVENT_TIMER_ADAPTER_EXT_CLK2,
	123	};
	124
	125	/* TIM_MEM_BUCKET */
	126	struct tim_mem_bucket {
	127	uint64_t first_chunk;
	128	union {
	129	uint64_t w1;
	130	struct {
	131	uint32_t nb_entry;
	132	uint8_t sbt:1;
	133	uint8_t hbt:1;
	134	uint8_t bsk:1;
	135	uint8_t rsvd:5;
	136	uint8_t lock;
	137	int16_t chunk_remainder;
	138	};
	139	};
	140	uint64_t current_chunk;
	141	uint64_t pad;
	142	} __rte_packed __rte_aligned(8);
	143
	144	struct tim_mem_entry {
	145	uint64_t w0;
	146	uint64_t wqe;
	147	} __rte_packed;
	148
	149	struct timvf_ctrl_reg {
	150	uint64_t rctrl0;
	151	uint64_t rctrl1;
	152	uint64_t rctrl2;
	153	uint8_t use_pmu;
	154	} __rte_packed;
	155
	156	struct timvf_ring;
	157
	158	typedef uint32_t (*bkt_id)(const uint32_t bkt_tcks, const uint32_t nb_bkts);
	159	typedef struct tim_mem_entry * (*refill_chunk)(
	160	struct tim_mem_bucket * const bkt,
	161	struct timvf_ring * const timr);
	162
	163	struct timvf_ring {
	164	bkt_id get_target_bkt;
	165	refill_chunk refill_chunk;
	166	struct rte_reciprocal_u64 fast_div;
	167	uint64_t ring_start_cyc;
	168	uint32_t nb_bkts;
	169	struct tim_mem_bucket *bkt;
	170	void *chunk_pool;
	171	uint64_t tck_int;
	172	volatile uint64_t tim_arm_cnt;
	173	uint64_t tck_nsec;
	174	void *vbar0;
	175	void *bkt_pos;
	176	uint64_t max_tout;
	177	uint64_t nb_chunks;
	178	enum timvf_clk_src clk_src;
	179	uint16_t tim_ring_id;
	180	} __rte_cache_aligned;
	181
182	static __rte_always_inline uint32_t
183	bkt_mod(const uint32_t rel_bkt, const uint32_t nb_bkts)
184	{
185	return rel_bkt % nb_bkts;
186	}
187
188	static __rte_always_inline uint32_t
189	bkt_and(uint32_t rel_bkt, uint32_t nb_bkts)
190	{
191	return rel_bkt & (nb_bkts - 1);
192	}
193
f67539c2 TL	194	uint8_t timvf_get_ring(void);
f67539c2 TL	195	void timvf_release_ring(uint8_t vfid);
11fdf7f2 TL	196	void *timvf_bar(uint8_t id, uint8_t bar);
	197	int timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags,
	198	uint32_t caps, const struct rte_event_timer_adapter_ops *ops,
	199	uint8_t enable_stats);
	200	uint16_t timvf_timer_cancel_burst(const struct rte_event_timer_adapter *adptr,
	201	struct rte_event_timer **tim, const uint16_t nb_timers);
	202	uint16_t timvf_timer_arm_burst_sp(const struct rte_event_timer_adapter *adptr,
	203	struct rte_event_timer **tim, const uint16_t nb_timers);
	204	uint16_t timvf_timer_arm_burst_sp_stats(
	205	const struct rte_event_timer_adapter *adptr,
	206	struct rte_event_timer **tim, const uint16_t nb_timers);
	207	uint16_t timvf_timer_arm_burst_mp(const struct rte_event_timer_adapter *adptr,
	208	struct rte_event_timer **tim, const uint16_t nb_timers);
	209	uint16_t timvf_timer_arm_burst_mp_stats(
	210	const struct rte_event_timer_adapter *adptr,
	211	struct rte_event_timer **tim, const uint16_t nb_timers);
	212	uint16_t timvf_timer_arm_tmo_brst(const struct rte_event_timer_adapter *adptr,
	213	struct rte_event_timer **tim, const uint64_t timeout_tick,
	214	const uint16_t nb_timers);
	215	uint16_t timvf_timer_arm_tmo_brst_stats(
	216	const struct rte_event_timer_adapter *adptr,
	217	struct rte_event_timer **tim, const uint64_t timeout_tick,
	218	const uint16_t nb_timers);
	219	void timvf_set_chunk_refill(struct timvf_ring * const timr, uint8_t use_fpa);
	220
	221	#endif /* __TIMVF_EVDEV_H__ */