1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* AF_XDP internal functions
3 * Copyright(c) 2018 Intel Corporation.
6 #ifndef _LINUX_XDP_SOCK_H
7 #define _LINUX_XDP_SOCK_H
9 #include <linux/workqueue.h>
10 #include <linux/if_xdp.h>
11 #include <linux/mutex.h>
12 #include <linux/spinlock.h>
19 struct xdp_umem_page
{
24 struct xdp_umem_fq_reuse
{
33 struct xdp_umem_page
*pages
;
38 struct user_struct
*user
;
39 unsigned long address
;
41 struct work_struct work
;
45 struct net_device
*dev
;
46 struct xdp_umem_fq_reuse
*fq_reuse
;
49 spinlock_t xsk_list_lock
;
50 struct list_head xsk_list
;
54 /* struct sock must be the first member of struct xdp_sock */
57 struct net_device
*dev
;
58 struct xdp_umem
*umem
;
59 struct list_head flush_node
;
62 /* Protects multiple processes in the control path */
64 struct xsk_queue
*tx ____cacheline_aligned_in_smp
;
65 struct list_head list
;
66 /* Mutual exclusion of NAPI TX thread and sendmsg error paths
67 * in the SKB destructor callback.
69 spinlock_t tx_completion_lock
;
70 /* Protects generic receive. */
76 #ifdef CONFIG_XDP_SOCKETS
77 int xsk_generic_rcv(struct xdp_sock
*xs
, struct xdp_buff
*xdp
);
78 int xsk_rcv(struct xdp_sock
*xs
, struct xdp_buff
*xdp
);
79 void xsk_flush(struct xdp_sock
*xs
);
80 bool xsk_is_setup_for_bpf_map(struct xdp_sock
*xs
);
81 /* Used from netdev driver */
82 bool xsk_umem_has_addrs(struct xdp_umem
*umem
, u32 cnt
);
83 u64
*xsk_umem_peek_addr(struct xdp_umem
*umem
, u64
*addr
);
84 void xsk_umem_discard_addr(struct xdp_umem
*umem
);
85 void xsk_umem_complete_tx(struct xdp_umem
*umem
, u32 nb_entries
);
86 bool xsk_umem_consume_tx(struct xdp_umem
*umem
, struct xdp_desc
*desc
);
87 void xsk_umem_consume_tx_done(struct xdp_umem
*umem
);
88 struct xdp_umem_fq_reuse
*xsk_reuseq_prepare(u32 nentries
);
89 struct xdp_umem_fq_reuse
*xsk_reuseq_swap(struct xdp_umem
*umem
,
90 struct xdp_umem_fq_reuse
*newq
);
91 void xsk_reuseq_free(struct xdp_umem_fq_reuse
*rq
);
92 struct xdp_umem
*xdp_get_umem_from_qid(struct net_device
*dev
, u16 queue_id
);
94 static inline char *xdp_umem_get_data(struct xdp_umem
*umem
, u64 addr
)
96 return umem
->pages
[addr
>> PAGE_SHIFT
].addr
+ (addr
& (PAGE_SIZE
- 1));
99 static inline dma_addr_t
xdp_umem_get_dma(struct xdp_umem
*umem
, u64 addr
)
101 return umem
->pages
[addr
>> PAGE_SHIFT
].dma
+ (addr
& (PAGE_SIZE
- 1));
104 /* Reuse-queue aware version of FILL queue helpers */
105 static inline bool xsk_umem_has_addrs_rq(struct xdp_umem
*umem
, u32 cnt
)
107 struct xdp_umem_fq_reuse
*rq
= umem
->fq_reuse
;
109 if (rq
->length
>= cnt
)
112 return xsk_umem_has_addrs(umem
, cnt
- rq
->length
);
115 static inline u64
*xsk_umem_peek_addr_rq(struct xdp_umem
*umem
, u64
*addr
)
117 struct xdp_umem_fq_reuse
*rq
= umem
->fq_reuse
;
120 return xsk_umem_peek_addr(umem
, addr
);
122 *addr
= rq
->handles
[rq
->length
- 1];
126 static inline void xsk_umem_discard_addr_rq(struct xdp_umem
*umem
)
128 struct xdp_umem_fq_reuse
*rq
= umem
->fq_reuse
;
131 xsk_umem_discard_addr(umem
);
136 static inline void xsk_umem_fq_reuse(struct xdp_umem
*umem
, u64 addr
)
138 struct xdp_umem_fq_reuse
*rq
= umem
->fq_reuse
;
140 rq
->handles
[rq
->length
++] = addr
;
143 static inline int xsk_generic_rcv(struct xdp_sock
*xs
, struct xdp_buff
*xdp
)
148 static inline int xsk_rcv(struct xdp_sock
*xs
, struct xdp_buff
*xdp
)
153 static inline void xsk_flush(struct xdp_sock
*xs
)
157 static inline bool xsk_is_setup_for_bpf_map(struct xdp_sock
*xs
)
162 static inline bool xsk_umem_has_addrs(struct xdp_umem
*umem
, u32 cnt
)
167 static inline u64
*xsk_umem_peek_addr(struct xdp_umem
*umem
, u64
*addr
)
172 static inline void xsk_umem_discard_addr(struct xdp_umem
*umem
)
176 static inline void xsk_umem_complete_tx(struct xdp_umem
*umem
, u32 nb_entries
)
180 static inline bool xsk_umem_consume_tx(struct xdp_umem
*umem
,
181 struct xdp_desc
*desc
)
186 static inline void xsk_umem_consume_tx_done(struct xdp_umem
*umem
)
190 static inline struct xdp_umem_fq_reuse
*xsk_reuseq_prepare(u32 nentries
)
195 static inline struct xdp_umem_fq_reuse
*xsk_reuseq_swap(
196 struct xdp_umem
*umem
,
197 struct xdp_umem_fq_reuse
*newq
)
201 static inline void xsk_reuseq_free(struct xdp_umem_fq_reuse
*rq
)
205 static inline struct xdp_umem
*xdp_get_umem_from_qid(struct net_device
*dev
,
211 static inline char *xdp_umem_get_data(struct xdp_umem
*umem
, u64 addr
)
216 static inline dma_addr_t
xdp_umem_get_dma(struct xdp_umem
*umem
, u64 addr
)
221 static inline bool xsk_umem_has_addrs_rq(struct xdp_umem
*umem
, u32 cnt
)
226 static inline u64
*xsk_umem_peek_addr_rq(struct xdp_umem
*umem
, u64
*addr
)
231 static inline void xsk_umem_discard_addr_rq(struct xdp_umem
*umem
)
235 static inline void xsk_umem_fq_reuse(struct xdp_umem
*umem
, u64 addr
)
239 #endif /* CONFIG_XDP_SOCKETS */
241 #endif /* _LINUX_XDP_SOCK_H */