[mirror_ubuntu-hirsute-kernel.git] / include / net / xdp.h

/* SPDX-License-Identifier: GPL-2.0-only */
/* include/net/xdp.h
 *
 * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
 */
#ifndef __LINUX_NET_XDP_H__
#define __LINUX_NET_XDP_H__

/**
 * DOC: XDP RX-queue information
 *
 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
 * level RX-ring queues.  It is information that is specific to how
 * the driver have configured a given RX-ring queue.
 *
 * Each xdp_buff frame received in the driver carry a (pointer)
 * reference to this xdp_rxq_info structure.  This provides the XDP
 * data-path read-access to RX-info for both kernel and bpf-side
 * (limited subset).
 *
 * For now, direct access is only safe while running in NAPI/softirq
 * context.  Contents is read-mostly and must not be updated during
 * driver NAPI/softirq poll.
 *
 * The driver usage API is a register and unregister API.
 *
 * The struct is not directly tied to the XDP prog.  A new XDP prog
 * can be attached as long as it doesn't change the underlying
 * RX-ring.  If the RX-ring does change significantly, the NIC driver
 * naturally need to stop the RX-ring before purging and reallocating
 * memory.  In that process the driver MUST call unregistor (which
 * also apply for driver shutdown and unload).  The register API is
 * also mandatory during RX-ring setup.
 */

enum xdp_mem_type {
	MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
	MEM_TYPE_PAGE_ORDER0,     /* Orig XDP full page model */
	MEM_TYPE_PAGE_POOL,
	MEM_TYPE_ZERO_COPY,
	MEM_TYPE_MAX,
};

/* XDP flags for ndo_xdp_xmit */
#define XDP_XMIT_FLUSH		(1U << 0)	/* doorbell signal consumer */
#define XDP_XMIT_FLAGS_MASK	XDP_XMIT_FLUSH

struct xdp_mem_info {
	u32 type; /* enum xdp_mem_type, but known size type */
	u32 id;
};

struct page_pool;

struct zero_copy_allocator {
	void (*free)(struct zero_copy_allocator *zca, unsigned long handle);
};

struct xdp_rxq_info {
	struct net_device *dev;
	u32 queue_index;
	u32 reg_state;
	struct xdp_mem_info mem;
} ____cacheline_aligned; /* perf critical, avoid false-sharing */

struct xdp_buff {
	void *data;
	void *data_end;
	void *data_meta;
	void *data_hard_start;
	unsigned long handle;
	struct xdp_rxq_info *rxq;
};

struct xdp_frame {
	void *data;
	u16 len;
	u16 headroom;
	u16 metasize;
	/* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
	 * while mem info is valid on remote CPU.
	 */
	struct xdp_mem_info mem;
	struct net_device *dev_rx; /* used by cpumap */
};

/* Clear kernel pointers in xdp_frame */
static inline void xdp_scrub_frame(struct xdp_frame *frame)
{
	frame->data = NULL;
	frame->dev_rx = NULL;
}

struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);

/* Convert xdp_buff to xdp_frame */
static inline
struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
{
	struct xdp_frame *xdp_frame;
	int metasize;
	int headroom;

	if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
		return xdp_convert_zc_to_xdp_frame(xdp);

	/* Assure headroom is available for storing info */
	headroom = xdp->data - xdp->data_hard_start;
	metasize = xdp->data - xdp->data_meta;
	metasize = metasize > 0 ? metasize : 0;
	if (unlikely((headroom - metasize) < sizeof(*xdp_frame)))
		return NULL;

	/* Store info in top of packet */
	xdp_frame = xdp->data_hard_start;

	xdp_frame->data = xdp->data;
	xdp_frame->len  = xdp->data_end - xdp->data;
	xdp_frame->headroom = headroom - sizeof(*xdp_frame);
	xdp_frame->metasize = metasize;

	/* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
	xdp_frame->mem = xdp->rxq->mem;

	return xdp_frame;
}

void xdp_return_frame(struct xdp_frame *xdpf);
void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
void xdp_return_buff(struct xdp_buff *xdp);

/* When sending xdp_frame into the network stack, then there is no
 * return point callback, which is needed to release e.g. DMA-mapping
 * resources with page_pool.  Thus, have explicit function to release
 * frame resources.
 */
void __xdp_release_frame(void *data, struct xdp_mem_info *mem);
static inline void xdp_release_frame(struct xdp_frame *xdpf)
{
	struct xdp_mem_info *mem = &xdpf->mem;

	/* Curr only page_pool needs this */
	if (mem->type == MEM_TYPE_PAGE_POOL)
		__xdp_release_frame(xdpf->data, mem);
}

int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
		     struct net_device *dev, u32 queue_index);
void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);
int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
			       enum xdp_mem_type type, void *allocator);
void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq);

/* Drivers not supporting XDP metadata can use this helper, which
 * rejects any room expansion for metadata as a result.
 */
static __always_inline void
xdp_set_data_meta_invalid(struct xdp_buff *xdp)
{
	xdp->data_meta = xdp->data + 1;
}

static __always_inline bool
xdp_data_meta_unsupported(const struct xdp_buff *xdp)
{
	return unlikely(xdp->data_meta > xdp->data);
}

struct xdp_attachment_info {
	struct bpf_prog *prog;
	u32 flags;
};

struct netdev_bpf;
int xdp_attachment_query(struct xdp_attachment_info *info,
			 struct netdev_bpf *bpf);
bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
			     struct netdev_bpf *bpf);
void xdp_attachment_setup(struct xdp_attachment_info *info,
			  struct netdev_bpf *bpf);

#endif /* __LINUX_NET_XDP_H__ */
Commit	Line	Data
ddc64d0a	1	/* SPDX-License-Identifier: GPL-2.0-only */
aecd67b6 JDB	2	/* include/net/xdp.h
	3	*
	4	* Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
aecd67b6 JDB	5	*/
	6	#ifndef __LINUX_NET_XDP_H__
	7	#define __LINUX_NET_XDP_H__
	8
	9	/**
	10	* DOC: XDP RX-queue information
	11	*
	12	* The XDP RX-queue info (xdp_rxq_info) is associated with the driver
	13	* level RX-ring queues. It is information that is specific to how
	14	* the driver have configured a given RX-ring queue.
	15	*
	16	* Each xdp_buff frame received in the driver carry a (pointer)
	17	* reference to this xdp_rxq_info structure. This provides the XDP
	18	* data-path read-access to RX-info for both kernel and bpf-side
	19	* (limited subset).
	20	*
	21	* For now, direct access is only safe while running in NAPI/softirq
	22	* context. Contents is read-mostly and must not be updated during
	23	* driver NAPI/softirq poll.
	24	*
	25	* The driver usage API is a register and unregister API.
	26	*
	27	* The struct is not directly tied to the XDP prog. A new XDP prog
	28	* can be attached as long as it doesn't change the underlying
	29	* RX-ring. If the RX-ring does change significantly, the NIC driver
	30	* naturally need to stop the RX-ring before purging and reallocating
	31	* memory. In that process the driver MUST call unregistor (which
	32	* also apply for driver shutdown and unload). The register API is
	33	* also mandatory during RX-ring setup.
	34	*/
	35
5ab073ff JDB	36	enum xdp_mem_type {
	37	MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
	38	MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
57d0a1c1	39	MEM_TYPE_PAGE_POOL,
02b55e56	40	MEM_TYPE_ZERO_COPY,
5ab073ff JDB	41	MEM_TYPE_MAX,
	42	};
	43
42b33468	44	/* XDP flags for ndo_xdp_xmit */
42b33468 JDB	45	#define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */
	46	#define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH
	47
5ab073ff JDB	48	struct xdp_mem_info {
5ab073ff JDB	49	u32 type; /* enum xdp_mem_type, but known size type */
8d5d8852	50	u32 id;
5ab073ff JDB	51	};
5ab073ff JDB	52
57d0a1c1 JDB	53	struct page_pool;
57d0a1c1 JDB	54
02b55e56 BT	55	struct zero_copy_allocator {
	56	void (free)(struct zero_copy_allocator zca, unsigned long handle);
	57	};
	58
aecd67b6 JDB	59	struct xdp_rxq_info {
	60	struct net_device *dev;
	61	u32 queue_index;
	62	u32 reg_state;
5ab073ff	63	struct xdp_mem_info mem;
aecd67b6 JDB	64	} ____cacheline_aligned; /* perf critical, avoid false-sharing */
aecd67b6 JDB	65
106ca27f JDB	66	struct xdp_buff {
	67	void *data;
	68	void *data_end;
	69	void *data_meta;
	70	void *data_hard_start;
02b55e56	71	unsigned long handle;
106ca27f JDB	72	struct xdp_rxq_info *rxq;
106ca27f JDB	73	};
5ab073ff	74
c0048cff JDB	75	struct xdp_frame {
	76	void *data;
	77	u16 len;
	78	u16 headroom;
	79	u16 metasize;
	80	/* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
	81	* while mem info is valid on remote CPU.
	82	*/
	83	struct xdp_mem_info mem;
70280ed9	84	struct net_device dev_rx; / used by cpumap */
c0048cff JDB	85	};
c0048cff JDB	86
a8d5b4ab TM	87	/* Clear kernel pointers in xdp_frame */
	88	static inline void xdp_scrub_frame(struct xdp_frame *frame)
	89	{
	90	frame->data = NULL;
	91	frame->dev_rx = NULL;
	92	}
	93
b0d1beef BT	94	struct xdp_frame xdp_convert_zc_to_xdp_frame(struct xdp_buff xdp);
b0d1beef BT	95
c0048cff JDB	96	/* Convert xdp_buff to xdp_frame */
	97	static inline
	98	struct xdp_frame convert_to_xdp_frame(struct xdp_buff xdp)
	99	{
	100	struct xdp_frame *xdp_frame;
	101	int metasize;
	102	int headroom;
	103
02b55e56	104	if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
b0d1beef	105	return xdp_convert_zc_to_xdp_frame(xdp);
02b55e56	106
c0048cff JDB	107	/* Assure headroom is available for storing info */
	108	headroom = xdp->data - xdp->data_hard_start;
	109	metasize = xdp->data - xdp->data_meta;
	110	metasize = metasize > 0 ? metasize : 0;
	111	if (unlikely((headroom - metasize) < sizeof(*xdp_frame)))
	112	return NULL;
	113
	114	/* Store info in top of packet */
	115	xdp_frame = xdp->data_hard_start;
	116
	117	xdp_frame->data = xdp->data;
	118	xdp_frame->len = xdp->data_end - xdp->data;
	119	xdp_frame->headroom = headroom - sizeof(*xdp_frame);
	120	xdp_frame->metasize = metasize;
	121
	122	/* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
	123	xdp_frame->mem = xdp->rxq->mem;
	124
	125	return xdp_frame;
	126	}
	127
03993094	128	void xdp_return_frame(struct xdp_frame *xdpf);
389ab7f0	129	void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
c497176c	130	void xdp_return_buff(struct xdp_buff *xdp);
5ab073ff	131
6bf071bf JDB	132	/* When sending xdp_frame into the network stack, then there is no
	133	* return point callback, which is needed to release e.g. DMA-mapping
	134	* resources with page_pool. Thus, have explicit function to release
	135	* frame resources.
	136	*/
	137	void __xdp_release_frame(void data, struct xdp_mem_info mem);
	138	static inline void xdp_release_frame(struct xdp_frame *xdpf)
	139	{
	140	struct xdp_mem_info *mem = &xdpf->mem;
	141
	142	/* Curr only page_pool needs this */
	143	if (mem->type == MEM_TYPE_PAGE_POOL)
	144	__xdp_release_frame(xdpf->data, mem);
	145	}
	146
aecd67b6 JDB	147	int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
	148	struct net_device *dev, u32 queue_index);
	149	void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
	150	void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
c0124f32	151	bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);
5ab073ff JDB	152	int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
5ab073ff JDB	153	enum xdp_mem_type type, void *allocator);
dce5bd61	154	void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq);
aecd67b6	155
106ca27f JDB	156	/* Drivers not supporting XDP metadata can use this helper, which
	157	* rejects any room expansion for metadata as a result.
	158	*/
	159	static __always_inline void
	160	xdp_set_data_meta_invalid(struct xdp_buff *xdp)
	161	{
	162	xdp->data_meta = xdp->data + 1;
	163	}
	164
	165	static __always_inline bool
	166	xdp_data_meta_unsupported(const struct xdp_buff *xdp)
	167	{
	168	return unlikely(xdp->data_meta > xdp->data);
	169	}
	170
05296620 JK	171	struct xdp_attachment_info {
	172	struct bpf_prog *prog;
	173	u32 flags;
	174	};
	175
	176	struct netdev_bpf;
	177	int xdp_attachment_query(struct xdp_attachment_info *info,
	178	struct netdev_bpf *bpf);
	179	bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
	180	struct netdev_bpf *bpf);
	181	void xdp_attachment_setup(struct xdp_attachment_info *info,
	182	struct netdev_bpf *bpf);
	183
aecd67b6	184	#endif /* __LINUX_NET_XDP_H__ */