[mirror_ubuntu-hirsute-kernel.git] / net / xdp / xsk_queue.h

/* SPDX-License-Identifier: GPL-2.0 */
/* XDP user-space ring structure
 * Copyright(c) 2018 Intel Corporation.
 */

#ifndef _LINUX_XSK_QUEUE_H
#define _LINUX_XSK_QUEUE_H

#include <linux/types.h>
#include <linux/if_xdp.h>
#include <net/xdp_sock.h>

#define RX_BATCH_SIZE 16
#define LAZY_UPDATE_THRESHOLD 128

struct xdp_ring {
	u32 producer ____cacheline_aligned_in_smp;
	u32 consumer ____cacheline_aligned_in_smp;
};

/* Used for the RX and TX queues for packets */
struct xdp_rxtx_ring {
	struct xdp_ring ptrs;
	struct xdp_desc desc[0] ____cacheline_aligned_in_smp;
};

/* Used for the fill and completion queues for buffers */
struct xdp_umem_ring {
	struct xdp_ring ptrs;
	u64 desc[0] ____cacheline_aligned_in_smp;
};

struct xsk_queue {
	u64 chunk_mask;
	u64 size;
	u32 ring_mask;
	u32 nentries;
	u32 prod_head;
	u32 prod_tail;
	u32 cons_head;
	u32 cons_tail;
	struct xdp_ring *ring;
	u64 invalid_descs;
};

/* Common functions operating for both RXTX and umem queues */

static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)
{
	return q ? q->invalid_descs : 0;
}

static inline u32 xskq_nb_avail(struct xsk_queue *q, u32 dcnt)
{
	u32 entries = q->prod_tail - q->cons_tail;

	if (entries == 0) {
		/* Refresh the local pointer */
		q->prod_tail = READ_ONCE(q->ring->producer);
		entries = q->prod_tail - q->cons_tail;
	}

	return (entries > dcnt) ? dcnt : entries;
}

static inline u32 xskq_nb_free(struct xsk_queue *q, u32 producer, u32 dcnt)
{
	u32 free_entries = q->nentries - (producer - q->cons_tail);

	if (free_entries >= dcnt)
		return free_entries;

	/* Refresh the local tail pointer */
	q->cons_tail = READ_ONCE(q->ring->consumer);
	return q->nentries - (producer - q->cons_tail);
}

/* UMEM queue */

static inline bool xskq_is_valid_addr(struct xsk_queue *q, u64 addr)
{
	if (addr >= q->size) {
		q->invalid_descs++;
		return false;
	}

	return true;
}

static inline u64 *xskq_validate_addr(struct xsk_queue *q, u64 *addr)
{
	while (q->cons_tail != q->cons_head) {
		struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
		unsigned int idx = q->cons_tail & q->ring_mask;

		*addr = READ_ONCE(ring->desc[idx]) & q->chunk_mask;
		if (xskq_is_valid_addr(q, *addr))
			return addr;

		q->cons_tail++;
	}

	return NULL;
}

static inline u64 *xskq_peek_addr(struct xsk_queue *q, u64 *addr)
{
	if (q->cons_tail == q->cons_head) {
		WRITE_ONCE(q->ring->consumer, q->cons_tail);
		q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);

		/* Order consumer and data */
		smp_rmb();
	}

	return xskq_validate_addr(q, addr);
}

static inline void xskq_discard_addr(struct xsk_queue *q)
{
	q->cons_tail++;
}

static inline int xskq_produce_addr(struct xsk_queue *q, u64 addr)
{
	struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;

	if (xskq_nb_free(q, q->prod_tail, 1) == 0)
		return -ENOSPC;

	ring->desc[q->prod_tail++ & q->ring_mask] = addr;

	/* Order producer and data */
	smp_wmb();

	WRITE_ONCE(q->ring->producer, q->prod_tail);
	return 0;
}

static inline int xskq_produce_addr_lazy(struct xsk_queue *q, u64 addr)
{
	struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;

	if (xskq_nb_free(q, q->prod_head, LAZY_UPDATE_THRESHOLD) == 0)
		return -ENOSPC;

	ring->desc[q->prod_head++ & q->ring_mask] = addr;
	return 0;
}

static inline void xskq_produce_flush_addr_n(struct xsk_queue *q,
					     u32 nb_entries)
{
	/* Order producer and data */
	smp_wmb();

	q->prod_tail += nb_entries;
	WRITE_ONCE(q->ring->producer, q->prod_tail);
}

static inline int xskq_reserve_addr(struct xsk_queue *q)
{
	if (xskq_nb_free(q, q->prod_head, 1) == 0)
		return -ENOSPC;

	q->prod_head++;
	return 0;
}

/* Rx/Tx queue */

static inline bool xskq_is_valid_desc(struct xsk_queue *q, struct xdp_desc *d)
{
	if (!xskq_is_valid_addr(q, d->addr))
		return false;

	if (((d->addr + d->len) & q->chunk_mask) !=
	    (d->addr & q->chunk_mask)) {
		q->invalid_descs++;
		return false;
	}

	return true;
}

static inline struct xdp_desc *xskq_validate_desc(struct xsk_queue *q,
						  struct xdp_desc *desc)
{
	while (q->cons_tail != q->cons_head) {
		struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
		unsigned int idx = q->cons_tail & q->ring_mask;

		*desc = READ_ONCE(ring->desc[idx]);
		if (xskq_is_valid_desc(q, desc))
			return desc;

		q->cons_tail++;
	}

	return NULL;
}

static inline struct xdp_desc *xskq_peek_desc(struct xsk_queue *q,
					      struct xdp_desc *desc)
{
	if (q->cons_tail == q->cons_head) {
		WRITE_ONCE(q->ring->consumer, q->cons_tail);
		q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);

		/* Order consumer and data */
		smp_rmb();
	}

	return xskq_validate_desc(q, desc);
}

static inline void xskq_discard_desc(struct xsk_queue *q)
{
	q->cons_tail++;
}

static inline int xskq_produce_batch_desc(struct xsk_queue *q,
					  u64 addr, u32 len)
{
	struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
	unsigned int idx;

	if (xskq_nb_free(q, q->prod_head, 1) == 0)
		return -ENOSPC;

	idx = (q->prod_head++) & q->ring_mask;
	ring->desc[idx].addr = addr;
	ring->desc[idx].len = len;

	return 0;
}

static inline void xskq_produce_flush_desc(struct xsk_queue *q)
{
	/* Order producer and data */
	smp_wmb();

	q->prod_tail = q->prod_head,
	WRITE_ONCE(q->ring->producer, q->prod_tail);
}

static inline bool xskq_full_desc(struct xsk_queue *q)
{
	return xskq_nb_avail(q, q->nentries) == q->nentries;
}

static inline bool xskq_empty_desc(struct xsk_queue *q)
{
	return xskq_nb_free(q, q->prod_tail, q->nentries) == q->nentries;
}

void xskq_set_umem(struct xsk_queue *q, u64 size, u64 chunk_mask);
struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
void xskq_destroy(struct xsk_queue *q_ops);

/* Executed by the core when the entire UMEM gets freed */
void xsk_reuseq_destroy(struct xdp_umem *umem);

#endif /* _LINUX_XSK_QUEUE_H */
Commit	Line	Data
dac09149 BT	1	/* SPDX-License-Identifier: GPL-2.0 */
dac09149 BT	2	/* XDP user-space ring structure
423f3832	3	* Copyright(c) 2018 Intel Corporation.
423f3832 MK	4	*/
	5
	6	#ifndef _LINUX_XSK_QUEUE_H
	7	#define _LINUX_XSK_QUEUE_H
	8
	9	#include <linux/types.h>
	10	#include <linux/if_xdp.h>
e61e62b9	11	#include <net/xdp_sock.h>
423f3832	12
c497176c	13	#define RX_BATCH_SIZE 16
ac98d8aa	14	#define LAZY_UPDATE_THRESHOLD 128
c497176c	15
b3a9e0be BT	16	struct xdp_ring {
	17	u32 producer ____cacheline_aligned_in_smp;
	18	u32 consumer ____cacheline_aligned_in_smp;
	19	};
	20
	21	/* Used for the RX and TX queues for packets */
	22	struct xdp_rxtx_ring {
	23	struct xdp_ring ptrs;
	24	struct xdp_desc desc[0] ____cacheline_aligned_in_smp;
	25	};
	26
	27	/* Used for the fill and completion queues for buffers */
	28	struct xdp_umem_ring {
	29	struct xdp_ring ptrs;
bbff2f32	30	u64 desc[0] ____cacheline_aligned_in_smp;
b3a9e0be BT	31	};
b3a9e0be BT	32
423f3832	33	struct xsk_queue {
93ee30f3 MK	34	u64 chunk_mask;
93ee30f3 MK	35	u64 size;
423f3832 MK	36	u32 ring_mask;
	37	u32 nentries;
	38	u32 prod_head;
	39	u32 prod_tail;
	40	u32 cons_head;
	41	u32 cons_tail;
	42	struct xdp_ring *ring;
	43	u64 invalid_descs;
	44	};
	45
c497176c BT	46	/* Common functions operating for both RXTX and umem queues */
c497176c BT	47
af75d9e0 MK	48	static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)
	49	{
	50	return q ? q->invalid_descs : 0;
	51	}
	52
c497176c BT	53	static inline u32 xskq_nb_avail(struct xsk_queue *q, u32 dcnt)
	54	{
	55	u32 entries = q->prod_tail - q->cons_tail;
	56
	57	if (entries == 0) {
	58	/* Refresh the local pointer */
	59	q->prod_tail = READ_ONCE(q->ring->producer);
	60	entries = q->prod_tail - q->cons_tail;
	61	}
	62
	63	return (entries > dcnt) ? dcnt : entries;
	64	}
	65
	66	static inline u32 xskq_nb_free(struct xsk_queue *q, u32 producer, u32 dcnt)
	67	{
20b52a75	68	u32 free_entries = q->nentries - (producer - q->cons_tail);
c497176c BT	69
	70	if (free_entries >= dcnt)
	71	return free_entries;
	72
	73	/* Refresh the local tail pointer */
	74	q->cons_tail = READ_ONCE(q->ring->consumer);
	75	return q->nentries - (producer - q->cons_tail);
	76	}
	77
	78	/* UMEM queue */
	79
bbff2f32	80	static inline bool xskq_is_valid_addr(struct xsk_queue *q, u64 addr)
c497176c	81	{
93ee30f3	82	if (addr >= q->size) {
c497176c BT	83	q->invalid_descs++;
	84	return false;
	85	}
bbff2f32	86
c497176c BT	87	return true;
	88	}
	89
bbff2f32	90	static inline u64 xskq_validate_addr(struct xsk_queue q, u64 *addr)
c497176c BT	91	{
	92	while (q->cons_tail != q->cons_head) {
	93	struct xdp_umem_ring ring = (struct xdp_umem_ring )q->ring;
	94	unsigned int idx = q->cons_tail & q->ring_mask;
	95
93ee30f3	96	*addr = READ_ONCE(ring->desc[idx]) & q->chunk_mask;
bbff2f32 BT	97	if (xskq_is_valid_addr(q, *addr))
bbff2f32 BT	98	return addr;
c497176c BT	99
	100	q->cons_tail++;
	101	}
	102
	103	return NULL;
	104	}
	105
bbff2f32	106	static inline u64 xskq_peek_addr(struct xsk_queue q, u64 *addr)
c497176c	107	{
c497176c BT	108	if (q->cons_tail == q->cons_head) {
	109	WRITE_ONCE(q->ring->consumer, q->cons_tail);
	110	q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);
	111
	112	/* Order consumer and data */
	113	smp_rmb();
c497176c BT	114	}
c497176c BT	115
bbff2f32	116	return xskq_validate_addr(q, addr);
c497176c BT	117	}
c497176c BT	118
bbff2f32	119	static inline void xskq_discard_addr(struct xsk_queue *q)
c497176c BT	120	{
c497176c BT	121	q->cons_tail++;
c497176c BT	122	}
c497176c BT	123
bbff2f32	124	static inline int xskq_produce_addr(struct xsk_queue *q, u64 addr)
35fcde7f MK	125	{
	126	struct xdp_umem_ring ring = (struct xdp_umem_ring )q->ring;
	127
20b52a75	128	if (xskq_nb_free(q, q->prod_tail, 1) == 0)
ac98d8aa MK	129	return -ENOSPC;
ac98d8aa MK	130
bbff2f32	131	ring->desc[q->prod_tail++ & q->ring_mask] = addr;
35fcde7f MK	132
	133	/* Order producer and data */
	134	smp_wmb();
	135
	136	WRITE_ONCE(q->ring->producer, q->prod_tail);
	137	return 0;
	138	}
	139
ac98d8aa MK	140	static inline int xskq_produce_addr_lazy(struct xsk_queue *q, u64 addr)
	141	{
	142	struct xdp_umem_ring ring = (struct xdp_umem_ring )q->ring;
	143
	144	if (xskq_nb_free(q, q->prod_head, LAZY_UPDATE_THRESHOLD) == 0)
	145	return -ENOSPC;
	146
	147	ring->desc[q->prod_head++ & q->ring_mask] = addr;
	148	return 0;
	149	}
	150
	151	static inline void xskq_produce_flush_addr_n(struct xsk_queue *q,
	152	u32 nb_entries)
	153	{
	154	/* Order producer and data */
	155	smp_wmb();
	156
	157	q->prod_tail += nb_entries;
	158	WRITE_ONCE(q->ring->producer, q->prod_tail);
	159	}
	160
bbff2f32	161	static inline int xskq_reserve_addr(struct xsk_queue *q)
35fcde7f MK	162	{
	163	if (xskq_nb_free(q, q->prod_head, 1) == 0)
	164	return -ENOSPC;
	165
	166	q->prod_head++;
	167	return 0;
	168	}
	169
	170	/* Rx/Tx queue */
	171
	172	static inline bool xskq_is_valid_desc(struct xsk_queue q, struct xdp_desc d)
	173	{
bbff2f32	174	if (!xskq_is_valid_addr(q, d->addr))
35fcde7f	175	return false;
35fcde7f	176
93ee30f3 MK	177	if (((d->addr + d->len) & q->chunk_mask) !=
93ee30f3 MK	178	(d->addr & q->chunk_mask)) {
35fcde7f MK	179	q->invalid_descs++;
	180	return false;
	181	}
	182
	183	return true;
	184	}
	185
	186	static inline struct xdp_desc xskq_validate_desc(struct xsk_queue q,
	187	struct xdp_desc *desc)
	188	{
	189	while (q->cons_tail != q->cons_head) {
	190	struct xdp_rxtx_ring ring = (struct xdp_rxtx_ring )q->ring;
	191	unsigned int idx = q->cons_tail & q->ring_mask;
	192
4e64c835 BT	193	*desc = READ_ONCE(ring->desc[idx]);
4e64c835 BT	194	if (xskq_is_valid_desc(q, desc))
35fcde7f	195	return desc;
35fcde7f MK	196
	197	q->cons_tail++;
	198	}
	199
	200	return NULL;
	201	}
	202
	203	static inline struct xdp_desc xskq_peek_desc(struct xsk_queue q,
	204	struct xdp_desc *desc)
	205	{
35fcde7f MK	206	if (q->cons_tail == q->cons_head) {
	207	WRITE_ONCE(q->ring->consumer, q->cons_tail);
	208	q->cons_head = q->cons_tail + xskq_nb_avail(q, RX_BATCH_SIZE);
	209
	210	/* Order consumer and data */
	211	smp_rmb();
35fcde7f MK	212	}
35fcde7f MK	213
4e64c835	214	return xskq_validate_desc(q, desc);
35fcde7f MK	215	}
	216
	217	static inline void xskq_discard_desc(struct xsk_queue *q)
	218	{
	219	q->cons_tail++;
35fcde7f	220	}
c497176c BT	221
c497176c BT	222	static inline int xskq_produce_batch_desc(struct xsk_queue *q,
bbff2f32	223	u64 addr, u32 len)
c497176c BT	224	{
	225	struct xdp_rxtx_ring ring = (struct xdp_rxtx_ring )q->ring;
	226	unsigned int idx;
	227
	228	if (xskq_nb_free(q, q->prod_head, 1) == 0)
	229	return -ENOSPC;
	230
	231	idx = (q->prod_head++) & q->ring_mask;
bbff2f32	232	ring->desc[idx].addr = addr;
c497176c	233	ring->desc[idx].len = len;
c497176c BT	234
	235	return 0;
	236	}
	237
	238	static inline void xskq_produce_flush_desc(struct xsk_queue *q)
	239	{
	240	/* Order producer and data */
	241	smp_wmb();
	242
	243	q->prod_tail = q->prod_head,
	244	WRITE_ONCE(q->ring->producer, q->prod_tail);
	245	}
	246
35fcde7f MK	247	static inline bool xskq_full_desc(struct xsk_queue *q)
35fcde7f MK	248	{
da60cf00	249	return xskq_nb_avail(q, q->nentries) == q->nentries;
35fcde7f MK	250	}
35fcde7f MK	251
c497176c BT	252	static inline bool xskq_empty_desc(struct xsk_queue *q)
c497176c BT	253	{
d24458e4	254	return xskq_nb_free(q, q->prod_tail, q->nentries) == q->nentries;
c497176c BT	255	}
c497176c BT	256
93ee30f3	257	void xskq_set_umem(struct xsk_queue *q, u64 size, u64 chunk_mask);
b9b6b68e	258	struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
c497176c	259	void xskq_destroy(struct xsk_queue *q_ops);
423f3832	260
f5bd9138 JK	261	/* Executed by the core when the entire UMEM gets freed */
	262	void xsk_reuseq_destroy(struct xdp_umem *umem);
	263
423f3832	264	#endif /* _LINUX_XSK_QUEUE_H */