[mirror_ubuntu-jammy-kernel.git] / net / core / sock_reuseport.c

// SPDX-License-Identifier: GPL-2.0
/*
 * To speed up listener socket lookup, create an array to store all sockets
 * listening on the same port.  This allows a decision to be made after finding
 * the first socket.  An optional BPF program can also be configured for
 * selecting the socket index from the array of available sockets.
 */

#include <net/sock_reuseport.h>
#include <linux/bpf.h>
#include <linux/idr.h>
#include <linux/filter.h>
#include <linux/rcupdate.h>

#define INIT_SOCKS 128

DEFINE_SPINLOCK(reuseport_lock);

static DEFINE_IDA(reuseport_ida);

static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks)
{
	unsigned int size = sizeof(struct sock_reuseport) +
		      sizeof(struct sock *) * max_socks;
	struct sock_reuseport *reuse = kzalloc(size, GFP_ATOMIC);

	if (!reuse)
		return NULL;

	reuse->max_socks = max_socks;

	RCU_INIT_POINTER(reuse->prog, NULL);
	return reuse;
}

int reuseport_alloc(struct sock *sk, bool bind_inany)
{
	struct sock_reuseport *reuse;
	int id, ret = 0;

	/* bh lock used since this function call may precede hlist lock in
	 * soft irq of receive path or setsockopt from process context
	 */
	spin_lock_bh(&reuseport_lock);

	/* Allocation attempts can occur concurrently via the setsockopt path
	 * and the bind/hash path.  Nothing to do when we lose the race.
	 */
	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
					  lockdep_is_held(&reuseport_lock));
	if (reuse) {
		/* Only set reuse->bind_inany if the bind_inany is true.
		 * Otherwise, it will overwrite the reuse->bind_inany
		 * which was set by the bind/hash path.
		 */
		if (bind_inany)
			reuse->bind_inany = bind_inany;
		goto out;
	}

	reuse = __reuseport_alloc(INIT_SOCKS);
	if (!reuse) {
		ret = -ENOMEM;
		goto out;
	}

	id = ida_alloc(&reuseport_ida, GFP_ATOMIC);
	if (id < 0) {
		kfree(reuse);
		ret = id;
		goto out;
	}

	reuse->reuseport_id = id;
	reuse->socks[0] = sk;
	reuse->num_socks = 1;
	reuse->bind_inany = bind_inany;
	rcu_assign_pointer(sk->sk_reuseport_cb, reuse);

out:
	spin_unlock_bh(&reuseport_lock);

	return ret;
}
EXPORT_SYMBOL(reuseport_alloc);

static struct sock_reuseport *reuseport_grow(struct sock_reuseport *reuse)
{
	struct sock_reuseport *more_reuse;
	u32 more_socks_size, i;

	more_socks_size = reuse->max_socks * 2U;
	if (more_socks_size > U16_MAX)
		return NULL;

	more_reuse = __reuseport_alloc(more_socks_size);
	if (!more_reuse)
		return NULL;

	more_reuse->num_socks = reuse->num_socks;
	more_reuse->prog = reuse->prog;
	more_reuse->reuseport_id = reuse->reuseport_id;
	more_reuse->bind_inany = reuse->bind_inany;

	memcpy(more_reuse->socks, reuse->socks,
	       reuse->num_socks * sizeof(struct sock *));
	more_reuse->synq_overflow_ts = READ_ONCE(reuse->synq_overflow_ts);

	for (i = 0; i < reuse->num_socks; ++i)
		rcu_assign_pointer(reuse->socks[i]->sk_reuseport_cb,
				   more_reuse);

	/* Note: we use kfree_rcu here instead of reuseport_free_rcu so
	 * that reuse and more_reuse can temporarily share a reference
	 * to prog.
	 */
	kfree_rcu(reuse, rcu);
	return more_reuse;
}

static void reuseport_free_rcu(struct rcu_head *head)
{
	struct sock_reuseport *reuse;

	reuse = container_of(head, struct sock_reuseport, rcu);
	sk_reuseport_prog_free(rcu_dereference_protected(reuse->prog, 1));
	ida_free(&reuseport_ida, reuse->reuseport_id);
	kfree(reuse);
}

/**
 *  reuseport_add_sock - Add a socket to the reuseport group of another.
 *  @sk:  New socket to add to the group.
 *  @sk2: Socket belonging to the existing reuseport group.
 *  @bind_inany: Whether or not the group is bound to a local INANY address.
 *
 *  May return ENOMEM and not add socket to group under memory pressure.
 */
int reuseport_add_sock(struct sock *sk, struct sock *sk2, bool bind_inany)
{
	struct sock_reuseport *old_reuse, *reuse;

	if (!rcu_access_pointer(sk2->sk_reuseport_cb)) {
		int err = reuseport_alloc(sk2, bind_inany);

		if (err)
			return err;
	}

	spin_lock_bh(&reuseport_lock);
	reuse = rcu_dereference_protected(sk2->sk_reuseport_cb,
					  lockdep_is_held(&reuseport_lock));
	old_reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
					     lockdep_is_held(&reuseport_lock));
	if (old_reuse && old_reuse->num_socks != 1) {
		spin_unlock_bh(&reuseport_lock);
		return -EBUSY;
	}

	if (reuse->num_socks == reuse->max_socks) {
		reuse = reuseport_grow(reuse);
		if (!reuse) {
			spin_unlock_bh(&reuseport_lock);
			return -ENOMEM;
		}
	}

	reuse->socks[reuse->num_socks] = sk;
	/* paired with smp_rmb() in reuseport_select_sock() */
	smp_wmb();
	reuse->num_socks++;
	rcu_assign_pointer(sk->sk_reuseport_cb, reuse);

	spin_unlock_bh(&reuseport_lock);

	if (old_reuse)
		call_rcu(&old_reuse->rcu, reuseport_free_rcu);
	return 0;
}
EXPORT_SYMBOL(reuseport_add_sock);

void reuseport_detach_sock(struct sock *sk)
{
	struct sock_reuseport *reuse;
	int i;

	spin_lock_bh(&reuseport_lock);
	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
					  lockdep_is_held(&reuseport_lock));

	/* Notify the bpf side. The sk may be added to a sockarray
	 * map. If so, sockarray logic will remove it from the map.
	 *
	 * Other bpf map types that work with reuseport, like sockmap,
	 * don't need an explicit callback from here. They override sk
	 * unhash/close ops to remove the sk from the map before we
	 * get to this point.
	 */
	bpf_sk_reuseport_detach(sk);

	rcu_assign_pointer(sk->sk_reuseport_cb, NULL);

	for (i = 0; i < reuse->num_socks; i++) {
		if (reuse->socks[i] == sk) {
			reuse->socks[i] = reuse->socks[reuse->num_socks - 1];
			reuse->num_socks--;
			if (reuse->num_socks == 0)
				call_rcu(&reuse->rcu, reuseport_free_rcu);
			break;
		}
	}
	spin_unlock_bh(&reuseport_lock);
}
EXPORT_SYMBOL(reuseport_detach_sock);

static struct sock *run_bpf_filter(struct sock_reuseport *reuse, u16 socks,
				   struct bpf_prog *prog, struct sk_buff *skb,
				   int hdr_len)
{
	struct sk_buff *nskb = NULL;
	u32 index;

	if (skb_shared(skb)) {
		nskb = skb_clone(skb, GFP_ATOMIC);
		if (!nskb)
			return NULL;
		skb = nskb;
	}

	/* temporarily advance data past protocol header */
	if (!pskb_pull(skb, hdr_len)) {
		kfree_skb(nskb);
		return NULL;
	}
	index = bpf_prog_run_save_cb(prog, skb);
	__skb_push(skb, hdr_len);

	consume_skb(nskb);

	if (index >= socks)
		return NULL;

	return reuse->socks[index];
}

/**
 *  reuseport_select_sock - Select a socket from an SO_REUSEPORT group.
 *  @sk: First socket in the group.
 *  @hash: When no BPF filter is available, use this hash to select.
 *  @skb: skb to run through BPF filter.
 *  @hdr_len: BPF filter expects skb data pointer at payload data.  If
 *    the skb does not yet point at the payload, this parameter represents
 *    how far the pointer needs to advance to reach the payload.
 *  Returns a socket that should receive the packet (or NULL on error).
 */
struct sock *reuseport_select_sock(struct sock *sk,
				   u32 hash,
				   struct sk_buff *skb,
				   int hdr_len)
{
	struct sock_reuseport *reuse;
	struct bpf_prog *prog;
	struct sock *sk2 = NULL;
	u16 socks;

	rcu_read_lock();
	reuse = rcu_dereference(sk->sk_reuseport_cb);

	/* if memory allocation failed or add call is not yet complete */
	if (!reuse)
		goto out;

	prog = rcu_dereference(reuse->prog);
	socks = READ_ONCE(reuse->num_socks);
	if (likely(socks)) {
		/* paired with smp_wmb() in reuseport_add_sock() */
		smp_rmb();

		if (!prog || !skb)
			goto select_by_hash;

		if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
			sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, hash);
		else
			sk2 = run_bpf_filter(reuse, socks, prog, skb, hdr_len);

select_by_hash:
		/* no bpf or invalid bpf result: fall back to hash usage */
		if (!sk2) {
			int i, j;

			i = j = reciprocal_scale(hash, socks);
			while (reuse->socks[i]->sk_state == TCP_ESTABLISHED) {
				i++;
				if (i >= reuse->num_socks)
					i = 0;
				if (i == j)
					goto out;
			}
			sk2 = reuse->socks[i];
		}
	}

out:
	rcu_read_unlock();
	return sk2;
}
EXPORT_SYMBOL(reuseport_select_sock);

int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog)
{
	struct sock_reuseport *reuse;
	struct bpf_prog *old_prog;

	if (sk_unhashed(sk) && sk->sk_reuseport) {
		int err = reuseport_alloc(sk, false);

		if (err)
			return err;
	} else if (!rcu_access_pointer(sk->sk_reuseport_cb)) {
		/* The socket wasn't bound with SO_REUSEPORT */
		return -EINVAL;
	}

	spin_lock_bh(&reuseport_lock);
	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
					  lockdep_is_held(&reuseport_lock));
	old_prog = rcu_dereference_protected(reuse->prog,
					     lockdep_is_held(&reuseport_lock));
	rcu_assign_pointer(reuse->prog, prog);
	spin_unlock_bh(&reuseport_lock);

	sk_reuseport_prog_free(old_prog);
	return 0;
}
EXPORT_SYMBOL(reuseport_attach_prog);

int reuseport_detach_prog(struct sock *sk)
{
	struct sock_reuseport *reuse;
	struct bpf_prog *old_prog;

	if (!rcu_access_pointer(sk->sk_reuseport_cb))
		return sk->sk_reuseport ? -ENOENT : -EINVAL;

	old_prog = NULL;
	spin_lock_bh(&reuseport_lock);
	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
					  lockdep_is_held(&reuseport_lock));
	old_prog = rcu_replace_pointer(reuse->prog, old_prog,
				       lockdep_is_held(&reuseport_lock));
	spin_unlock_bh(&reuseport_lock);

	if (!old_prog)
		return -ENOENT;

	sk_reuseport_prog_free(old_prog);
	return 0;
}
EXPORT_SYMBOL(reuseport_detach_prog);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
ef456144 CG	2	/*
	3	* To speed up listener socket lookup, create an array to store all sockets
	4	* listening on the same port. This allows a decision to be made after finding
538950a1 CG	5	* the first socket. An optional BPF program can also be configured for
538950a1 CG	6	* selecting the socket index from the array of available sockets.
ef456144 CG	7	*/
	8
	9	#include <net/sock_reuseport.h>
538950a1	10	#include <linux/bpf.h>
736b4602	11	#include <linux/idr.h>
8217ca65	12	#include <linux/filter.h>
ef456144 CG	13	#include <linux/rcupdate.h>
	14
	15	#define INIT_SOCKS 128
	16
736b4602 MKL	17	DEFINE_SPINLOCK(reuseport_lock);
736b4602 MKL	18
736b4602 MKL	19	static DEFINE_IDA(reuseport_ida);
736b4602 MKL	20
822f9bb1	21	static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks)
ef456144	22	{
822f9bb1	23	unsigned int size = sizeof(struct sock_reuseport) +
ef456144 CG	24	sizeof(struct sock ) max_socks;
	25	struct sock_reuseport *reuse = kzalloc(size, GFP_ATOMIC);
	26
	27	if (!reuse)
	28	return NULL;
	29
	30	reuse->max_socks = max_socks;
	31
538950a1	32	RCU_INIT_POINTER(reuse->prog, NULL);
ef456144 CG	33	return reuse;
	34	}
	35
2dbb9b9e	36	int reuseport_alloc(struct sock *sk, bool bind_inany)
ef456144 CG	37	{
ef456144 CG	38	struct sock_reuseport *reuse;
035ff358	39	int id, ret = 0;
ef456144 CG	40
	41	/* bh lock used since this function call may precede hlist lock in
	42	* soft irq of receive path or setsockopt from process context
	43	*/
	44	spin_lock_bh(&reuseport_lock);
1b5f962e CG	45
	46	/* Allocation attempts can occur concurrently via the setsockopt path
	47	* and the bind/hash path. Nothing to do when we lose the race.
	48	*/
2dbb9b9e MKL	49	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
	50	lockdep_is_held(&reuseport_lock));
	51	if (reuse) {
	52	/* Only set reuse->bind_inany if the bind_inany is true.
	53	* Otherwise, it will overwrite the reuse->bind_inany
	54	* which was set by the bind/hash path.
	55	*/
	56	if (bind_inany)
	57	reuse->bind_inany = bind_inany;
1b5f962e	58	goto out;
2dbb9b9e	59	}
1b5f962e	60
ef456144 CG	61	reuse = __reuseport_alloc(INIT_SOCKS);
ef456144 CG	62	if (!reuse) {
035ff358 JS	63	ret = -ENOMEM;
035ff358 JS	64	goto out;
ef456144 CG	65	}
ef456144 CG	66
035ff358 JS	67	id = ida_alloc(&reuseport_ida, GFP_ATOMIC);
	68	if (id < 0) {
	69	kfree(reuse);
	70	ret = id;
	71	goto out;
	72	}
	73
	74	reuse->reuseport_id = id;
ef456144 CG	75	reuse->socks[0] = sk;
ef456144 CG	76	reuse->num_socks = 1;
2dbb9b9e	77	reuse->bind_inany = bind_inany;
ef456144 CG	78	rcu_assign_pointer(sk->sk_reuseport_cb, reuse);
ef456144 CG	79
1b5f962e	80	out:
ef456144 CG	81	spin_unlock_bh(&reuseport_lock);
ef456144 CG	82
035ff358	83	return ret;
ef456144 CG	84	}
	85	EXPORT_SYMBOL(reuseport_alloc);
	86
	87	static struct sock_reuseport reuseport_grow(struct sock_reuseport reuse)
	88	{
	89	struct sock_reuseport *more_reuse;
	90	u32 more_socks_size, i;
	91
	92	more_socks_size = reuse->max_socks * 2U;
	93	if (more_socks_size > U16_MAX)
	94	return NULL;
	95
	96	more_reuse = __reuseport_alloc(more_socks_size);
	97	if (!more_reuse)
	98	return NULL;
	99
ef456144	100	more_reuse->num_socks = reuse->num_socks;
538950a1	101	more_reuse->prog = reuse->prog;
736b4602	102	more_reuse->reuseport_id = reuse->reuseport_id;
2dbb9b9e	103	more_reuse->bind_inany = reuse->bind_inany;
ef456144 CG	104
	105	memcpy(more_reuse->socks, reuse->socks,
	106	reuse->num_socks * sizeof(struct sock *));
40a1227e	107	more_reuse->synq_overflow_ts = READ_ONCE(reuse->synq_overflow_ts);
ef456144 CG	108
	109	for (i = 0; i < reuse->num_socks; ++i)
	110	rcu_assign_pointer(reuse->socks[i]->sk_reuseport_cb,
	111	more_reuse);
	112
538950a1 CG	113	/* Note: we use kfree_rcu here instead of reuseport_free_rcu so
	114	* that reuse and more_reuse can temporarily share a reference
	115	* to prog.
	116	*/
ef456144 CG	117	kfree_rcu(reuse, rcu);
	118	return more_reuse;
	119	}
	120
4db428a7 ED	121	static void reuseport_free_rcu(struct rcu_head *head)
	122	{
	123	struct sock_reuseport *reuse;
	124
	125	reuse = container_of(head, struct sock_reuseport, rcu);
8217ca65	126	sk_reuseport_prog_free(rcu_dereference_protected(reuse->prog, 1));
035ff358	127	ida_free(&reuseport_ida, reuse->reuseport_id);
4db428a7 ED	128	kfree(reuse);
	129	}
	130
ef456144 CG	131	/**
	132	* reuseport_add_sock - Add a socket to the reuseport group of another.
	133	* @sk: New socket to add to the group.
	134	* @sk2: Socket belonging to the existing reuseport group.
37f3c421 BVA	135	* @bind_inany: Whether or not the group is bound to a local INANY address.
37f3c421 BVA	136	*
ef456144 CG	137	* May return ENOMEM and not add socket to group under memory pressure.
ef456144 CG	138	*/
2dbb9b9e	139	int reuseport_add_sock(struct sock sk, struct sock sk2, bool bind_inany)
ef456144	140	{
4db428a7	141	struct sock_reuseport old_reuse, reuse;
ef456144	142
b4ace4f1	143	if (!rcu_access_pointer(sk2->sk_reuseport_cb)) {
2dbb9b9e	144	int err = reuseport_alloc(sk2, bind_inany);
b4ace4f1 CG	145
	146	if (err)
	147	return err;
	148	}
	149
ef456144 CG	150	spin_lock_bh(&reuseport_lock);
ef456144 CG	151	reuse = rcu_dereference_protected(sk2->sk_reuseport_cb,
4db428a7 ED	152	lockdep_is_held(&reuseport_lock));
	153	old_reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
	154	lockdep_is_held(&reuseport_lock));
	155	if (old_reuse && old_reuse->num_socks != 1) {
	156	spin_unlock_bh(&reuseport_lock);
	157	return -EBUSY;
	158	}
ef456144 CG	159
	160	if (reuse->num_socks == reuse->max_socks) {
	161	reuse = reuseport_grow(reuse);
	162	if (!reuse) {
	163	spin_unlock_bh(&reuseport_lock);
	164	return -ENOMEM;
	165	}
	166	}
	167
	168	reuse->socks[reuse->num_socks] = sk;
	169	/* paired with smp_rmb() in reuseport_select_sock() */
	170	smp_wmb();
	171	reuse->num_socks++;
	172	rcu_assign_pointer(sk->sk_reuseport_cb, reuse);
	173
	174	spin_unlock_bh(&reuseport_lock);
	175
4db428a7 ED	176	if (old_reuse)
4db428a7 ED	177	call_rcu(&old_reuse->rcu, reuseport_free_rcu);
ef456144 CG	178	return 0;
ef456144 CG	179	}
76c6d988	180	EXPORT_SYMBOL(reuseport_add_sock);
ef456144 CG	181
	182	void reuseport_detach_sock(struct sock *sk)
	183	{
	184	struct sock_reuseport *reuse;
	185	int i;
	186
	187	spin_lock_bh(&reuseport_lock);
	188	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
	189	lockdep_is_held(&reuseport_lock));
5dc4c4b7	190
035ff358 JS	191	/* Notify the bpf side. The sk may be added to a sockarray
	192	* map. If so, sockarray logic will remove it from the map.
	193	*
	194	* Other bpf map types that work with reuseport, like sockmap,
	195	* don't need an explicit callback from here. They override sk
	196	* unhash/close ops to remove the sk from the map before we
	197	* get to this point.
5dc4c4b7	198	*/
035ff358	199	bpf_sk_reuseport_detach(sk);
5dc4c4b7	200
ef456144 CG	201	rcu_assign_pointer(sk->sk_reuseport_cb, NULL);
	202
	203	for (i = 0; i < reuse->num_socks; i++) {
	204	if (reuse->socks[i] == sk) {
	205	reuse->socks[i] = reuse->socks[reuse->num_socks - 1];
	206	reuse->num_socks--;
	207	if (reuse->num_socks == 0)
538950a1	208	call_rcu(&reuse->rcu, reuseport_free_rcu);
ef456144 CG	209	break;
	210	}
	211	}
	212	spin_unlock_bh(&reuseport_lock);
	213	}
	214	EXPORT_SYMBOL(reuseport_detach_sock);
	215
8217ca65 MKL	216	static struct sock run_bpf_filter(struct sock_reuseport reuse, u16 socks,
	217	struct bpf_prog prog, struct sk_buff skb,
	218	int hdr_len)
538950a1 CG	219	{
	220	struct sk_buff *nskb = NULL;
	221	u32 index;
	222
	223	if (skb_shared(skb)) {
	224	nskb = skb_clone(skb, GFP_ATOMIC);
	225	if (!nskb)
	226	return NULL;
	227	skb = nskb;
	228	}
	229
	230	/* temporarily advance data past protocol header */
	231	if (!pskb_pull(skb, hdr_len)) {
00ce3a15	232	kfree_skb(nskb);
538950a1 CG	233	return NULL;
	234	}
	235	index = bpf_prog_run_save_cb(prog, skb);
	236	__skb_push(skb, hdr_len);
	237
	238	consume_skb(nskb);
	239
	240	if (index >= socks)
	241	return NULL;
	242
	243	return reuse->socks[index];
	244	}
	245
ef456144 CG	246	/**
	247	* reuseport_select_sock - Select a socket from an SO_REUSEPORT group.
	248	* @sk: First socket in the group.
538950a1 CG	249	* @hash: When no BPF filter is available, use this hash to select.
	250	* @skb: skb to run through BPF filter.
	251	* @hdr_len: BPF filter expects skb data pointer at payload data. If
	252	* the skb does not yet point at the payload, this parameter represents
	253	* how far the pointer needs to advance to reach the payload.
ef456144 CG	254	* Returns a socket that should receive the packet (or NULL on error).
ef456144 CG	255	*/
538950a1 CG	256	struct sock reuseport_select_sock(struct sock sk,
	257	u32 hash,
	258	struct sk_buff *skb,
	259	int hdr_len)
ef456144 CG	260	{
ef456144 CG	261	struct sock_reuseport *reuse;
538950a1	262	struct bpf_prog *prog;
ef456144 CG	263	struct sock *sk2 = NULL;
	264	u16 socks;
	265
	266	rcu_read_lock();
	267	reuse = rcu_dereference(sk->sk_reuseport_cb);
	268
	269	/* if memory allocation failed or add call is not yet complete */
	270	if (!reuse)
	271	goto out;
	272
538950a1	273	prog = rcu_dereference(reuse->prog);
ef456144 CG	274	socks = READ_ONCE(reuse->num_socks);
	275	if (likely(socks)) {
	276	/* paired with smp_wmb() in reuseport_add_sock() */
	277	smp_rmb();
	278
8217ca65 MKL	279	if (!prog \|\| !skb)
	280	goto select_by_hash;
	281
	282	if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
	283	sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, hash);
	284	else
	285	sk2 = run_bpf_filter(reuse, socks, prog, skb, hdr_len);
e94a62f5	286
8217ca65	287	select_by_hash:
e94a62f5	288	/* no bpf or invalid bpf result: fall back to hash usage */
acdcecc6 WB	289	if (!sk2) {
	290	int i, j;
	291
	292	i = j = reciprocal_scale(hash, socks);
	293	while (reuse->socks[i]->sk_state == TCP_ESTABLISHED) {
	294	i++;
	295	if (i >= reuse->num_socks)
	296	i = 0;
	297	if (i == j)
	298	goto out;
	299	}
	300	sk2 = reuse->socks[i];
	301	}
ef456144 CG	302	}
	303
	304	out:
	305	rcu_read_unlock();
	306	return sk2;
	307	}
	308	EXPORT_SYMBOL(reuseport_select_sock);
538950a1	309
8217ca65	310	int reuseport_attach_prog(struct sock sk, struct bpf_prog prog)
538950a1 CG	311	{
	312	struct sock_reuseport *reuse;
	313	struct bpf_prog *old_prog;
	314
8217ca65 MKL	315	if (sk_unhashed(sk) && sk->sk_reuseport) {
	316	int err = reuseport_alloc(sk, false);
	317
	318	if (err)
	319	return err;
	320	} else if (!rcu_access_pointer(sk->sk_reuseport_cb)) {
	321	/* The socket wasn't bound with SO_REUSEPORT */
	322	return -EINVAL;
	323	}
	324
538950a1 CG	325	spin_lock_bh(&reuseport_lock);
	326	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
	327	lockdep_is_held(&reuseport_lock));
	328	old_prog = rcu_dereference_protected(reuse->prog,
	329	lockdep_is_held(&reuseport_lock));
	330	rcu_assign_pointer(reuse->prog, prog);
	331	spin_unlock_bh(&reuseport_lock);
	332
8217ca65 MKL	333	sk_reuseport_prog_free(old_prog);
8217ca65 MKL	334	return 0;
538950a1 CG	335	}
538950a1 CG	336	EXPORT_SYMBOL(reuseport_attach_prog);
99f3a064 MKL	337
	338	int reuseport_detach_prog(struct sock *sk)
	339	{
	340	struct sock_reuseport *reuse;
	341	struct bpf_prog *old_prog;
	342
	343	if (!rcu_access_pointer(sk->sk_reuseport_cb))
	344	return sk->sk_reuseport ? -ENOENT : -EINVAL;
	345
	346	old_prog = NULL;
	347	spin_lock_bh(&reuseport_lock);
	348	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
	349	lockdep_is_held(&reuseport_lock));
e3f0d761 PM	350	old_prog = rcu_replace_pointer(reuse->prog, old_prog,
e3f0d761 PM	351	lockdep_is_held(&reuseport_lock));
99f3a064 MKL	352	spin_unlock_bh(&reuseport_lock);
	353
	354	if (!old_prog)
	355	return -ENOENT;
	356
	357	sk_reuseport_prog_free(old_prog);
	358	return 0;
	359	}
	360	EXPORT_SYMBOL(reuseport_detach_prog);