[mirror_frr.git] / bgpd / bgp_keepalives.c

// SPDX-License-Identifier: GPL-2.0-or-later
/* BGP Keepalives.
 * Implements a producer thread to generate BGP keepalives for peers.
 * Copyright (C) 2017 Cumulus Networks, Inc.
 * Quentin Young
 */

/* clang-format off */
#include <zebra.h>
#include <pthread.h>		// for pthread_mutex_lock, pthread_mutex_unlock

#include "frr_pthread.h"        // for frr_pthread
#include "hash.h"		// for hash, hash_clean, hash_create_size...
#include "log.h"		// for zlog_debug
#include "memory.h"		// for MTYPE_TMP, XFREE, XCALLOC, XMALLOC
#include "monotime.h"		// for monotime, monotime_since

#include "bgpd/bgpd.h"          // for peer, PEER_THREAD_KEEPALIVES_ON, peer...
#include "bgpd/bgp_debug.h"	// for bgp_debug_neighbor_events
#include "bgpd/bgp_packet.h"	// for bgp_keepalive_send
#include "bgpd/bgp_keepalives.h"
/* clang-format on */

DEFINE_MTYPE_STATIC(BGPD, BGP_PKAT, "Peer KeepAlive Timer");
DEFINE_MTYPE_STATIC(BGPD, BGP_COND, "BGP Peer pthread Conditional");
DEFINE_MTYPE_STATIC(BGPD, BGP_MUTEX, "BGP Peer pthread Mutex");

/*
 * Peer KeepAlive Timer.
 * Associates a peer with the time of its last keepalive.
 */
struct pkat {
	/* the peer to send keepalives to */
	struct peer *peer;
	/* absolute time of last keepalive sent */
	struct timeval last;
};

/* List of peers we are sending keepalives for, and associated mutex. */
static pthread_mutex_t *peerhash_mtx;
static pthread_cond_t *peerhash_cond;
static struct hash *peerhash;

static struct pkat *pkat_new(struct peer *peer)
{
	struct pkat *pkat = XMALLOC(MTYPE_BGP_PKAT, sizeof(struct pkat));
	pkat->peer = peer;
	monotime(&pkat->last);
	return pkat;
}

static void pkat_del(void *pkat)
{
	XFREE(MTYPE_BGP_PKAT, pkat);
}


/*
 * Callback for hash_iterate. Determines if a peer needs a keepalive and if so,
 * generates and sends it.
 *
 * For any given peer, if the elapsed time since its last keepalive exceeds its
 * configured keepalive timer, a keepalive is sent to the peer and its
 * last-sent time is reset. Additionally, If the elapsed time does not exceed
 * the configured keepalive timer, but the time until the next keepalive is due
 * is within a hardcoded tolerance, a keepalive is sent as if the configured
 * timer was exceeded. Doing this helps alleviate nanosecond sleeps between
 * ticks by grouping together peers who are due for keepalives at roughly the
 * same time. This tolerance value is arbitrarily chosen to be 100ms.
 *
 * In addition, this function calculates the maximum amount of time that the
 * keepalive thread can sleep before another tick needs to take place. This is
 * equivalent to shortest time until a keepalive is due for any one peer.
 *
 * @return maximum time to wait until next update (0 if infinity)
 */
static void peer_process(struct hash_bucket *hb, void *arg)
{
	struct pkat *pkat = hb->data;

	struct timeval *next_update = arg;

	static struct timeval elapsed;  // elapsed time since keepalive
	static struct timeval ka = {0}; // peer->v_keepalive as a timeval
	static struct timeval diff;     // ka - elapsed

	static const struct timeval tolerance = {0, 100000};

	uint32_t v_ka = atomic_load_explicit(&pkat->peer->v_keepalive,
					     memory_order_relaxed);

	/* 0 keepalive timer means no keepalives */
	if (v_ka == 0)
		return;

	/* calculate elapsed time since last keepalive */
	monotime_since(&pkat->last, &elapsed);

	/* calculate difference between elapsed time and configured time */
	ka.tv_sec = v_ka;
	timersub(&ka, &elapsed, &diff);

	int send_keepalive =
		elapsed.tv_sec >= ka.tv_sec || timercmp(&diff, &tolerance, <);

	if (send_keepalive) {
		if (bgp_debug_keepalive(pkat->peer))
			zlog_debug("%s [FSM] Timer (keepalive timer expire)",
				   pkat->peer->host);

		bgp_keepalive_send(pkat->peer);
		monotime(&pkat->last);
		memset(&elapsed, 0, sizeof(elapsed));
		diff = ka;
	}

	/* if calculated next update for this peer < current delay, use it */
	if (next_update->tv_sec < 0 || timercmp(&diff, next_update, <))
		*next_update = diff;
}

static bool peer_hash_cmp(const void *f, const void *s)
{
	const struct pkat *p1 = f;
	const struct pkat *p2 = s;

	return p1->peer == p2->peer;
}

static unsigned int peer_hash_key(const void *arg)
{
	const struct pkat *pkat = arg;
	return (uintptr_t)pkat->peer;
}

/* Cleanup handler / deinitializer. */
static void bgp_keepalives_finish(void *arg)
{
	if (peerhash) {
		hash_clean(peerhash, pkat_del);
		hash_free(peerhash);
	}

	peerhash = NULL;

	pthread_mutex_unlock(peerhash_mtx);
	pthread_mutex_destroy(peerhash_mtx);
	pthread_cond_destroy(peerhash_cond);

	XFREE(MTYPE_BGP_MUTEX, peerhash_mtx);
	XFREE(MTYPE_BGP_COND, peerhash_cond);
}

/*
 * Entry function for peer keepalive generation pthread.
 */
void *bgp_keepalives_start(void *arg)
{
	struct frr_pthread *fpt = arg;
	fpt->master->owner = pthread_self();

	struct timeval currtime = {0, 0};
	struct timeval aftertime = {0, 0};
	struct timeval next_update = {0, 0};
	struct timespec next_update_ts = {0, 0};

	/*
	 * The RCU mechanism for each pthread is initialized in a "locked"
	 * state. That's ok for pthreads using the frr_pthread,
	 * thread_fetch event loop, because that event loop unlocks regularly.
	 * For foreign pthreads, the lock needs to be unlocked so that the
	 * background rcu pthread can run.
	 */
	rcu_read_unlock();

	peerhash_mtx = XCALLOC(MTYPE_BGP_MUTEX, sizeof(pthread_mutex_t));
	peerhash_cond = XCALLOC(MTYPE_BGP_COND, sizeof(pthread_cond_t));

	/* initialize mutex */
	pthread_mutex_init(peerhash_mtx, NULL);

	/* use monotonic clock with condition variable */
	pthread_condattr_t attrs;
	pthread_condattr_init(&attrs);
	pthread_condattr_setclock(&attrs, CLOCK_MONOTONIC);
	pthread_cond_init(peerhash_cond, &attrs);
	pthread_condattr_destroy(&attrs);

	/*
	 * We are not using normal FRR pthread mechanics and are
	 * not using fpt_run
	 */
	frr_pthread_set_name(fpt);

	/* initialize peer hashtable */
	peerhash = hash_create_size(2048, peer_hash_key, peer_hash_cmp, NULL);
	pthread_mutex_lock(peerhash_mtx);

	/* register cleanup handler */
	pthread_cleanup_push(&bgp_keepalives_finish, NULL);

	/* notify anybody waiting on us that we are done starting up */
	frr_pthread_notify_running(fpt);

	while (atomic_load_explicit(&fpt->running, memory_order_relaxed)) {
		if (peerhash->count > 0)
			pthread_cond_timedwait(peerhash_cond, peerhash_mtx,
					       &next_update_ts);
		else
			while (peerhash->count == 0
			       && atomic_load_explicit(&fpt->running,
						       memory_order_relaxed))
				pthread_cond_wait(peerhash_cond, peerhash_mtx);

		monotime(&currtime);

		next_update.tv_sec = -1;

		hash_iterate(peerhash, peer_process, &next_update);
		if (next_update.tv_sec == -1)
			memset(&next_update, 0, sizeof(next_update));

		monotime_since(&currtime, &aftertime);

		timeradd(&currtime, &next_update, &next_update);
		TIMEVAL_TO_TIMESPEC(&next_update, &next_update_ts);
	}

	/* clean up */
	pthread_cleanup_pop(1);

	return NULL;
}

/* --- thread external functions ------------------------------------------- */

void bgp_keepalives_on(struct peer *peer)
{
	if (CHECK_FLAG(peer->thread_flags, PEER_THREAD_KEEPALIVES_ON))
		return;

	struct frr_pthread *fpt = bgp_pth_ka;
	assert(fpt->running);

	/* placeholder bucket data to use for fast key lookups */
	static struct pkat holder = {0};

	/*
	 * We need to ensure that bgp_keepalives_init was called first
	 */
	assert(peerhash_mtx);

	frr_with_mutex (peerhash_mtx) {
		holder.peer = peer;
		if (!hash_lookup(peerhash, &holder)) {
			struct pkat *pkat = pkat_new(peer);
			(void)hash_get(peerhash, pkat, hash_alloc_intern);
			peer_lock(peer);
		}
		SET_FLAG(peer->thread_flags, PEER_THREAD_KEEPALIVES_ON);
		/* Force the keepalive thread to wake up */
		pthread_cond_signal(peerhash_cond);
	}
}

void bgp_keepalives_off(struct peer *peer)
{
	if (!CHECK_FLAG(peer->thread_flags, PEER_THREAD_KEEPALIVES_ON))
		return;

	struct frr_pthread *fpt = bgp_pth_ka;
	assert(fpt->running);

	/* placeholder bucket data to use for fast key lookups */
	static struct pkat holder = {0};

	/*
	 * We need to ensure that bgp_keepalives_init was called first
	 */
	assert(peerhash_mtx);

	frr_with_mutex (peerhash_mtx) {
		holder.peer = peer;
		struct pkat *res = hash_release(peerhash, &holder);
		if (res) {
			pkat_del(res);
			peer_unlock(peer);
		}
		UNSET_FLAG(peer->thread_flags, PEER_THREAD_KEEPALIVES_ON);
	}
}

int bgp_keepalives_stop(struct frr_pthread *fpt, void **result)
{
	assert(fpt->running);

	frr_with_mutex (peerhash_mtx) {
		atomic_store_explicit(&fpt->running, false,
				      memory_order_relaxed);
		pthread_cond_signal(peerhash_cond);
	}

	pthread_join(fpt->thread, result);
	return 0;
}
Commit	Line	Data
acddc0ed	1	// SPDX-License-Identifier: GPL-2.0-or-later
5c0c651c QY	2	/* BGP Keepalives.
	3	* Implements a producer thread to generate BGP keepalives for peers.
	4	* Copyright (C) 2017 Cumulus Networks, Inc.
	5	* Quentin Young
03014d48	6	*/
5c0c651c	7
6ee8ea1c	8	/* clang-format off */
03014d48	9	#include <zebra.h>
6ee8ea1c	10	#include <pthread.h> // for pthread_mutex_lock, pthread_mutex_unlock
03014d48	11
6ee8ea1c QY	12	#include "frr_pthread.h" // for frr_pthread
	13	#include "hash.h" // for hash, hash_clean, hash_create_size...
	14	#include "log.h" // for zlog_debug
	15	#include "memory.h" // for MTYPE_TMP, XFREE, XCALLOC, XMALLOC
	16	#include "monotime.h" // for monotime, monotime_since
03014d48	17
6ee8ea1c QY	18	#include "bgpd/bgpd.h" // for peer, PEER_THREAD_KEEPALIVES_ON, peer...
	19	#include "bgpd/bgp_debug.h" // for bgp_debug_neighbor_events
	20	#include "bgpd/bgp_packet.h" // for bgp_keepalive_send
03014d48	21	#include "bgpd/bgp_keepalives.h"
6ee8ea1c	22	/* clang-format on */
03014d48	23
19a713be DS	24	DEFINE_MTYPE_STATIC(BGPD, BGP_PKAT, "Peer KeepAlive Timer");
	25	DEFINE_MTYPE_STATIC(BGPD, BGP_COND, "BGP Peer pthread Conditional");
	26	DEFINE_MTYPE_STATIC(BGPD, BGP_MUTEX, "BGP Peer pthread Mutex");
	27
a715eab3	28	/*
03014d48 QY	29	* Peer KeepAlive Timer.
	30	* Associates a peer with the time of its last keepalive.
	31	*/
	32	struct pkat {
a715eab3	33	/* the peer to send keepalives to */
03014d48	34	struct peer *peer;
a715eab3	35	/* absolute time of last keepalive sent */
03014d48 QY	36	struct timeval last;
	37	};
	38
	39	/* List of peers we are sending keepalives for, and associated mutex. */
bd8b71e4 QY	40	static pthread_mutex_t *peerhash_mtx;
	41	static pthread_cond_t *peerhash_cond;
	42	static struct hash *peerhash;
03014d48	43
03014d48 QY	44	static struct pkat pkat_new(struct peer peer)
03014d48 QY	45	{
19a713be	46	struct pkat *pkat = XMALLOC(MTYPE_BGP_PKAT, sizeof(struct pkat));
03014d48 QY	47	pkat->peer = peer;
	48	monotime(&pkat->last);
	49	return pkat;
	50	}
	51
	52	static void pkat_del(void *pkat)
	53	{
19a713be	54	XFREE(MTYPE_BGP_PKAT, pkat);
03014d48	55	}
03014d48	56
bd8b71e4	57
03014d48	58	/*
424ab01d QY	59	* Callback for hash_iterate. Determines if a peer needs a keepalive and if so,
424ab01d QY	60	* generates and sends it.
03014d48 QY	61	*
	62	* For any given peer, if the elapsed time since its last keepalive exceeds its
	63	* configured keepalive timer, a keepalive is sent to the peer and its
	64	* last-sent time is reset. Additionally, If the elapsed time does not exceed
	65	* the configured keepalive timer, but the time until the next keepalive is due
	66	* is within a hardcoded tolerance, a keepalive is sent as if the configured
	67	* timer was exceeded. Doing this helps alleviate nanosecond sleeps between
	68	* ticks by grouping together peers who are due for keepalives at roughly the
	69	* same time. This tolerance value is arbitrarily chosen to be 100ms.
	70	*
	71	* In addition, this function calculates the maximum amount of time that the
	72	* keepalive thread can sleep before another tick needs to take place. This is
	73	* equivalent to shortest time until a keepalive is due for any one peer.
	74	*
	75	* @return maximum time to wait until next update (0 if infinity)
	76	*/
e3b78da8	77	static void peer_process(struct hash_bucket hb, void arg)
03014d48	78	{
bd8b71e4 QY	79	struct pkat *pkat = hb->data;
	80
	81	struct timeval *next_update = arg;
03014d48	82
03014d48 QY	83	static struct timeval elapsed; // elapsed time since keepalive
	84	static struct timeval ka = {0}; // peer->v_keepalive as a timeval
	85	static struct timeval diff; // ka - elapsed
	86
2b64873d	87	static const struct timeval tolerance = {0, 100000};
03014d48	88
bfc18a02 QY	89	uint32_t v_ka = atomic_load_explicit(&pkat->peer->v_keepalive,
	90	memory_order_relaxed);
	91
	92	/* 0 keepalive timer means no keepalives */
	93	if (v_ka == 0)
	94	return;
	95
a715eab3	96	/* calculate elapsed time since last keepalive */
bd8b71e4	97	monotime_since(&pkat->last, &elapsed);
03014d48	98
a715eab3	99	/* calculate difference between elapsed time and configured time */
bfc18a02	100	ka.tv_sec = v_ka;
bd8b71e4	101	timersub(&ka, &elapsed, &diff);
03014d48	102
bd8b71e4 QY	103	int send_keepalive =
bd8b71e4 QY	104	elapsed.tv_sec >= ka.tv_sec \|\| timercmp(&diff, &tolerance, <);
03014d48	105
bd8b71e4	106	if (send_keepalive) {
3ffec403	107	if (bgp_debug_keepalive(pkat->peer))
bd8b71e4 QY	108	zlog_debug("%s [FSM] Timer (keepalive timer expire)",
bd8b71e4 QY	109	pkat->peer->host);
03014d48	110
bd8b71e4 QY	111	bgp_keepalive_send(pkat->peer);
bd8b71e4 QY	112	monotime(&pkat->last);
6006b807	113	memset(&elapsed, 0, sizeof(elapsed));
a715eab3	114	diff = ka;
03014d48 QY	115	}
03014d48 QY	116
a715eab3	117	/* if calculated next update for this peer < current delay, use it */
2ccf91b1	118	if (next_update->tv_sec < 0 \|\| timercmp(&diff, next_update, <))
bd8b71e4 QY	119	*next_update = diff;
	120	}
	121
74df8d6d	122	static bool peer_hash_cmp(const void f, const void s)
bd8b71e4 QY	123	{
	124	const struct pkat *p1 = f;
	125	const struct pkat *p2 = s;
74df8d6d	126
bd8b71e4 QY	127	return p1->peer == p2->peer;
	128	}
	129
d8b87afe	130	static unsigned int peer_hash_key(const void *arg)
bd8b71e4	131	{
d8b87afe	132	const struct pkat *pkat = arg;
bd8b71e4	133	return (uintptr_t)pkat->peer;
03014d48 QY	134	}
03014d48 QY	135
a715eab3	136	/* Cleanup handler / deinitializer. */
b72b6f4f	137	static void bgp_keepalives_finish(void *arg)
419dfe6a	138	{
bd8b71e4 QY	139	if (peerhash) {
	140	hash_clean(peerhash, pkat_del);
	141	hash_free(peerhash);
	142	}
419dfe6a	143
bd8b71e4	144	peerhash = NULL;
419dfe6a	145
bd8b71e4 QY	146	pthread_mutex_unlock(peerhash_mtx);
	147	pthread_mutex_destroy(peerhash_mtx);
	148	pthread_cond_destroy(peerhash_cond);
419dfe6a	149
19a713be DS	150	XFREE(MTYPE_BGP_MUTEX, peerhash_mtx);
19a713be DS	151	XFREE(MTYPE_BGP_COND, peerhash_cond);
419dfe6a QY	152	}
419dfe6a QY	153
a715eab3	154	/*
419dfe6a	155	* Entry function for peer keepalive generation pthread.
419dfe6a	156	*/
b72b6f4f	157	void bgp_keepalives_start(void arg)
419dfe6a	158	{
a715eab3 QY	159	struct frr_pthread *fpt = arg;
	160	fpt->master->owner = pthread_self();
	161
419dfe6a	162	struct timeval currtime = {0, 0};
bd8b71e4	163	struct timeval aftertime = {0, 0};
419dfe6a QY	164	struct timeval next_update = {0, 0};
	165	struct timespec next_update_ts = {0, 0};
	166
85ba04f3 MS	167	/*
	168	* The RCU mechanism for each pthread is initialized in a "locked"
	169	* state. That's ok for pthreads using the frr_pthread,
	170	* thread_fetch event loop, because that event loop unlocks regularly.
	171	* For foreign pthreads, the lock needs to be unlocked so that the
	172	* background rcu pthread can run.
	173	*/
	174	rcu_read_unlock();
	175
19a713be DS	176	peerhash_mtx = XCALLOC(MTYPE_BGP_MUTEX, sizeof(pthread_mutex_t));
19a713be DS	177	peerhash_cond = XCALLOC(MTYPE_BGP_COND, sizeof(pthread_cond_t));
a715eab3 QY	178
	179	/* initialize mutex */
	180	pthread_mutex_init(peerhash_mtx, NULL);
	181
	182	/* use monotonic clock with condition variable */
	183	pthread_condattr_t attrs;
	184	pthread_condattr_init(&attrs);
	185	pthread_condattr_setclock(&attrs, CLOCK_MONOTONIC);
	186	pthread_cond_init(peerhash_cond, &attrs);
	187	pthread_condattr_destroy(&attrs);
	188
c80bedb8 DS	189	/*
	190	* We are not using normal FRR pthread mechanics and are
	191	* not using fpt_run
	192	*/
	193	frr_pthread_set_name(fpt);
a9198bc1	194
a715eab3 QY	195	/* initialize peer hashtable */
a715eab3 QY	196	peerhash = hash_create_size(2048, peer_hash_key, peer_hash_cmp, NULL);
bd8b71e4	197	pthread_mutex_lock(peerhash_mtx);
03014d48	198
a715eab3	199	/* register cleanup handler */
b72b6f4f	200	pthread_cleanup_push(&bgp_keepalives_finish, NULL);
03014d48	201
a715eab3 QY	202	/* notify anybody waiting on us that we are done starting up */
a715eab3 QY	203	frr_pthread_notify_running(fpt);
03014d48	204
a715eab3	205	while (atomic_load_explicit(&fpt->running, memory_order_relaxed)) {
bd8b71e4 QY	206	if (peerhash->count > 0)
bd8b71e4 QY	207	pthread_cond_timedwait(peerhash_cond, peerhash_mtx,
03014d48 QY	208	&next_update_ts);
03014d48 QY	209	else
bd8b71e4	210	while (peerhash->count == 0
a715eab3 QY	211	&& atomic_load_explicit(&fpt->running,
a715eab3 QY	212	memory_order_relaxed))
bd8b71e4	213	pthread_cond_wait(peerhash_cond, peerhash_mtx);
03014d48 QY	214
03014d48 QY	215	monotime(&currtime);
bd8b71e4 QY	216
	217	next_update.tv_sec = -1;
	218
	219	hash_iterate(peerhash, peer_process, &next_update);
	220	if (next_update.tv_sec == -1)
6006b807	221	memset(&next_update, 0, sizeof(next_update));
bd8b71e4 QY	222
	223	monotime_since(&currtime, &aftertime);
	224
03014d48 QY	225	timeradd(&currtime, &next_update, &next_update);
	226	TIMEVAL_TO_TIMESPEC(&next_update, &next_update_ts);
	227	}
	228
a715eab3	229	/* clean up */
03014d48 QY	230	pthread_cleanup_pop(1);
	231
	232	return NULL;
	233	}
	234
	235	/* --- thread external functions ------------------------------------------- */
	236
b72b6f4f	237	void bgp_keepalives_on(struct peer *peer)
03014d48	238	{
096476dd QY	239	if (CHECK_FLAG(peer->thread_flags, PEER_THREAD_KEEPALIVES_ON))
	240	return;
	241
1ac267a2	242	struct frr_pthread *fpt = bgp_pth_ka;
a715eab3 QY	243	assert(fpt->running);
a715eab3 QY	244
bd8b71e4 QY	245	/* placeholder bucket data to use for fast key lookups */
	246	static struct pkat holder = {0};
	247
68ede9c4 DS	248	/*
	249	* We need to ensure that bgp_keepalives_init was called first
	250	*/
	251	assert(peerhash_mtx);
934af458	252
cb1991af	253	frr_with_mutex (peerhash_mtx) {
bd8b71e4 QY	254	holder.peer = peer;
	255	if (!hash_lookup(peerhash, &holder)) {
	256	struct pkat *pkat = pkat_new(peer);
8e3aae66	257	(void)hash_get(peerhash, pkat, hash_alloc_intern);
bd8b71e4 QY	258	peer_lock(peer);
bd8b71e4 QY	259	}
49507a6f	260	SET_FLAG(peer->thread_flags, PEER_THREAD_KEEPALIVES_ON);
8c9d306c SB	261	/* Force the keepalive thread to wake up */
8c9d306c SB	262	pthread_cond_signal(peerhash_cond);
03014d48	263	}
03014d48 QY	264	}
03014d48 QY	265
b72b6f4f	266	void bgp_keepalives_off(struct peer *peer)
03014d48	267	{
096476dd QY	268	if (!CHECK_FLAG(peer->thread_flags, PEER_THREAD_KEEPALIVES_ON))
	269	return;
	270
1ac267a2	271	struct frr_pthread *fpt = bgp_pth_ka;
a715eab3 QY	272	assert(fpt->running);
a715eab3 QY	273
bd8b71e4 QY	274	/* placeholder bucket data to use for fast key lookups */
bd8b71e4 QY	275	static struct pkat holder = {0};
49507a6f	276
68ede9c4 DS	277	/*
	278	* We need to ensure that bgp_keepalives_init was called first
	279	*/
	280	assert(peerhash_mtx);
934af458	281
cb1991af	282	frr_with_mutex (peerhash_mtx) {
bd8b71e4 QY	283	holder.peer = peer;
	284	struct pkat *res = hash_release(peerhash, &holder);
	285	if (res) {
	286	pkat_del(res);
	287	peer_unlock(peer);
	288	}
49507a6f	289	UNSET_FLAG(peer->thread_flags, PEER_THREAD_KEEPALIVES_ON);
03014d48	290	}
03014d48 QY	291	}
03014d48 QY	292
a715eab3	293	int bgp_keepalives_stop(struct frr_pthread fpt, void *result)
0ca8b79f	294	{
a715eab3 QY	295	assert(fpt->running);
a715eab3 QY	296
8c9d306c SB	297	frr_with_mutex (peerhash_mtx) {
	298	atomic_store_explicit(&fpt->running, false,
	299	memory_order_relaxed);
	300	pthread_cond_signal(peerhash_cond);
	301	}
a715eab3	302
0ca8b79f QY	303	pthread_join(fpt->thread, result);
	304	return 0;
	305	}