1 /* BGP-4 Finite State Machine
2 * From RFC1771 [A Border Gateway Protocol 4 (BGP-4)]
3 * Copyright (C) 1996, 97, 98 Kunihiro Ishiguro
5 * This file is part of GNU Zebra.
7 * GNU Zebra is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2, or (at your option) any
12 * GNU Zebra is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License for more details.
17 * You should have received a copy of the GNU General Public License along
18 * with this program; see the file COPYING; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26 #include "sockunion.h"
33 #include "workqueue.h"
37 #include "lib_errors.h"
40 #include "bgpd/bgpd.h"
41 #include "bgpd/bgp_attr.h"
42 #include "bgpd/bgp_debug.h"
43 #include "bgpd/bgp_errors.h"
44 #include "bgpd/bgp_fsm.h"
45 #include "bgpd/bgp_packet.h"
46 #include "bgpd/bgp_network.h"
47 #include "bgpd/bgp_route.h"
48 #include "bgpd/bgp_dump.h"
49 #include "bgpd/bgp_open.h"
50 #include "bgpd/bgp_advertise.h"
51 #include "bgpd/bgp_updgrp.h"
52 #include "bgpd/bgp_nht.h"
53 #include "bgpd/bgp_bfd.h"
54 #include "bgpd/bgp_memory.h"
55 #include "bgpd/bgp_keepalives.h"
56 #include "bgpd/bgp_io.h"
57 #include "bgpd/bgp_zebra.h"
59 DEFINE_HOOK(peer_backward_transition
, (struct peer
* peer
), (peer
))
60 DEFINE_HOOK(peer_status_changed
, (struct peer
* peer
), (peer
))
61 extern const char *get_afi_safi_str(afi_t afi
, safi_t safi
, bool for_json
);
62 /* Definition of display strings corresponding to FSM events. This should be
63 * kept consistent with the events defined in bgpd.h
65 static const char *const bgp_event_str
[] = {
69 "TCP_connection_open",
70 "TCP_connection_closed",
71 "TCP_connection_open_failed",
73 "ConnectRetry_timer_expired",
75 "KeepAlive_timer_expired",
76 "Receive_OPEN_message",
77 "Receive_KEEPALIVE_message",
78 "Receive_UPDATE_message",
79 "Receive_NOTIFICATION_message",
83 /* BGP FSM (finite state machine) has three types of functions. Type
84 one is thread functions. Type two is event functions. Type three
85 is FSM functions. Timer functions are set by bgp_timer_set
88 /* BGP event function. */
89 int bgp_event(struct thread
*);
91 /* BGP thread functions. */
92 static int bgp_start_timer(struct thread
*);
93 static int bgp_connect_timer(struct thread
*);
94 static int bgp_holdtime_timer(struct thread
*);
96 /* BGP FSM functions. */
97 static int bgp_start(struct peer
*);
99 /* Register peer with NHT */
100 static int bgp_peer_reg_with_nht(struct peer
*peer
)
104 if (peer
->sort
== BGP_PEER_EBGP
&& peer
->ttl
== BGP_DEFAULT_TTL
105 && !CHECK_FLAG(peer
->flags
, PEER_FLAG_DISABLE_CONNECTED_CHECK
)
106 && !CHECK_FLAG(peer
->bgp
->flags
, BGP_FLAG_DISABLE_NH_CONNECTED_CHK
))
109 return bgp_find_or_add_nexthop(
110 peer
->bgp
, peer
->bgp
, family2afi(peer
->su
.sa
.sa_family
),
111 NULL
, peer
, connected
);
114 static void peer_xfer_stats(struct peer
*peer_dst
, struct peer
*peer_src
)
116 /* Copy stats over. These are only the pre-established state stats */
117 peer_dst
->open_in
+= peer_src
->open_in
;
118 peer_dst
->open_out
+= peer_src
->open_out
;
119 peer_dst
->keepalive_in
+= peer_src
->keepalive_in
;
120 peer_dst
->keepalive_out
+= peer_src
->keepalive_out
;
121 peer_dst
->notify_in
+= peer_src
->notify_in
;
122 peer_dst
->notify_out
+= peer_src
->notify_out
;
123 peer_dst
->dynamic_cap_in
+= peer_src
->dynamic_cap_in
;
124 peer_dst
->dynamic_cap_out
+= peer_src
->dynamic_cap_out
;
127 static struct peer
*peer_xfer_conn(struct peer
*from_peer
)
133 enum bgp_fsm_status status
, pstatus
;
134 enum bgp_fsm_events last_evt
, last_maj_evt
;
136 assert(from_peer
!= NULL
);
138 peer
= from_peer
->doppelganger
;
140 if (!peer
|| !CHECK_FLAG(peer
->flags
, PEER_FLAG_CONFIG_NODE
))
144 * Let's check that we are not going to loose known configuration
145 * state based upon doppelganger rules.
147 FOREACH_AFI_SAFI (afi
, safi
) {
148 if (from_peer
->afc
[afi
][safi
] != peer
->afc
[afi
][safi
]) {
150 EC_BGP_DOPPELGANGER_CONFIG
,
151 "from_peer->afc[%d][%d] is not the same as what we are overwriting",
157 if (bgp_debug_neighbor_events(peer
))
158 zlog_debug("%s: peer transfer %p fd %d -> %p fd %d)",
159 from_peer
->host
, from_peer
, from_peer
->fd
, peer
,
162 bgp_writes_off(peer
);
164 bgp_writes_off(from_peer
);
165 bgp_reads_off(from_peer
);
168 * Before exchanging FD remove doppelganger from
169 * keepalive peer hash. It could be possible conf peer
170 * fd is set to -1. If blocked on lock then keepalive
171 * thread can access peer pointer with fd -1.
173 bgp_keepalives_off(from_peer
);
175 BGP_TIMER_OFF(peer
->t_routeadv
);
176 BGP_TIMER_OFF(peer
->t_connect
);
177 BGP_TIMER_OFF(peer
->t_connect_check_r
);
178 BGP_TIMER_OFF(peer
->t_connect_check_w
);
179 BGP_TIMER_OFF(from_peer
->t_routeadv
);
180 BGP_TIMER_OFF(from_peer
->t_connect
);
181 BGP_TIMER_OFF(from_peer
->t_connect_check_r
);
182 BGP_TIMER_OFF(from_peer
->t_connect_check_w
);
183 BGP_TIMER_OFF(from_peer
->t_process_packet
);
186 * At this point in time, it is possible that there are packets pending
187 * on various buffers. Those need to be transferred or dropped,
188 * otherwise we'll get spurious failures during session establishment.
190 frr_with_mutex(&peer
->io_mtx
, &from_peer
->io_mtx
) {
192 peer
->fd
= from_peer
->fd
;
195 stream_fifo_clean(peer
->ibuf
);
196 stream_fifo_clean(peer
->obuf
);
199 * this should never happen, since bgp_process_packet() is the
200 * only task that sets and unsets the current packet and it
201 * runs in our pthread.
206 "[%s] Dropping pending packet on connection transfer:",
208 /* there used to be a bgp_packet_dump call here, but
209 * that's extremely confusing since there's no way to
210 * identify the packet in MRT dumps or BMP as dropped
211 * due to connection transfer.
213 stream_free(peer
->curr
);
217 // copy each packet from old peer's output queue to new peer
218 while (from_peer
->obuf
->head
)
219 stream_fifo_push(peer
->obuf
,
220 stream_fifo_pop(from_peer
->obuf
));
222 // copy each packet from old peer's input queue to new peer
223 while (from_peer
->ibuf
->head
)
224 stream_fifo_push(peer
->ibuf
,
225 stream_fifo_pop(from_peer
->ibuf
));
227 ringbuf_wipe(peer
->ibuf_work
);
228 ringbuf_copy(peer
->ibuf_work
, from_peer
->ibuf_work
,
229 ringbuf_remain(from_peer
->ibuf_work
));
232 peer
->as
= from_peer
->as
;
233 peer
->v_holdtime
= from_peer
->v_holdtime
;
234 peer
->v_keepalive
= from_peer
->v_keepalive
;
235 peer
->v_routeadv
= from_peer
->v_routeadv
;
236 peer
->v_gr_restart
= from_peer
->v_gr_restart
;
237 peer
->cap
= from_peer
->cap
;
238 status
= peer
->status
;
239 pstatus
= peer
->ostatus
;
240 last_evt
= peer
->last_event
;
241 last_maj_evt
= peer
->last_major_event
;
242 peer
->status
= from_peer
->status
;
243 peer
->ostatus
= from_peer
->ostatus
;
244 peer
->last_event
= from_peer
->last_event
;
245 peer
->last_major_event
= from_peer
->last_major_event
;
246 from_peer
->status
= status
;
247 from_peer
->ostatus
= pstatus
;
248 from_peer
->last_event
= last_evt
;
249 from_peer
->last_major_event
= last_maj_evt
;
250 peer
->remote_id
= from_peer
->remote_id
;
251 peer
->last_reset
= from_peer
->last_reset
;
253 peer
->peer_gr_present_state
= from_peer
->peer_gr_present_state
;
254 peer
->peer_gr_new_status_flag
= from_peer
->peer_gr_new_status_flag
;
255 bgp_peer_gr_flags_update(peer
);
257 BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(peer
->bgp
,
260 if (bgp_peer_gr_mode_get(peer
) == PEER_DISABLE
) {
262 UNSET_FLAG(peer
->sflags
, PEER_STATUS_NSF_MODE
);
264 if (CHECK_FLAG(peer
->sflags
, PEER_STATUS_NSF_WAIT
)) {
269 if (from_peer
->hostname
!= NULL
) {
270 if (peer
->hostname
) {
271 XFREE(MTYPE_BGP_PEER_HOST
, peer
->hostname
);
272 peer
->hostname
= NULL
;
275 peer
->hostname
= from_peer
->hostname
;
276 from_peer
->hostname
= NULL
;
279 if (from_peer
->domainname
!= NULL
) {
280 if (peer
->domainname
) {
281 XFREE(MTYPE_BGP_PEER_HOST
, peer
->domainname
);
282 peer
->domainname
= NULL
;
285 peer
->domainname
= from_peer
->domainname
;
286 from_peer
->domainname
= NULL
;
289 FOREACH_AFI_SAFI (afi
, safi
) {
290 peer
->af_flags
[afi
][safi
] = from_peer
->af_flags
[afi
][safi
];
291 peer
->af_sflags
[afi
][safi
] = from_peer
->af_sflags
[afi
][safi
];
292 peer
->af_cap
[afi
][safi
] = from_peer
->af_cap
[afi
][safi
];
293 peer
->afc_nego
[afi
][safi
] = from_peer
->afc_nego
[afi
][safi
];
294 peer
->afc_adv
[afi
][safi
] = from_peer
->afc_adv
[afi
][safi
];
295 peer
->afc_recv
[afi
][safi
] = from_peer
->afc_recv
[afi
][safi
];
296 peer
->orf_plist
[afi
][safi
] = from_peer
->orf_plist
[afi
][safi
];
299 if (bgp_getsockname(peer
) < 0) {
302 "%%bgp_getsockname() failed for %s peer %s fd %d (from_peer fd %d)",
303 (CHECK_FLAG(peer
->sflags
, PEER_STATUS_ACCEPT_PEER
)
306 peer
->host
, peer
->fd
, from_peer
->fd
);
311 if (from_peer
->status
> Active
) {
312 if (bgp_getsockname(from_peer
) < 0) {
315 "%%bgp_getsockname() failed for %s from_peer %s fd %d (peer fd %d)",
317 (CHECK_FLAG(from_peer
->sflags
,
318 PEER_STATUS_ACCEPT_PEER
)
321 from_peer
->host
, from_peer
->fd
, peer
->fd
);
328 // Note: peer_xfer_stats() must be called with I/O turned OFF
330 peer_xfer_stats(peer
, from_peer
);
332 /* Register peer for NHT. This is to allow RAs to be enabled when
333 * needed, even on a passive connection.
335 bgp_peer_reg_with_nht(peer
);
339 thread_add_timer_msec(bm
->master
, bgp_process_packet
, peer
, 0,
340 &peer
->t_process_packet
);
345 /* Hook function called after bgp event is occered. And vty's
346 neighbor command invoke this function after making neighbor
348 void bgp_timer_set(struct peer
*peer
)
350 switch (peer
->status
) {
352 /* First entry point of peer's finite state machine. In Idle
353 status start timer is on unless peer is shutdown or peer is
354 inactive. All other timer must be turned off */
355 if (BGP_PEER_START_SUPPRESSED(peer
) || !peer_active(peer
)
356 || (peer
->bgp
->inst_type
!= BGP_INSTANCE_TYPE_VIEW
&&
357 peer
->bgp
->vrf_id
== VRF_UNKNOWN
)) {
358 BGP_TIMER_OFF(peer
->t_start
);
360 BGP_TIMER_ON(peer
->t_start
, bgp_start_timer
,
363 BGP_TIMER_OFF(peer
->t_connect
);
364 BGP_TIMER_OFF(peer
->t_holdtime
);
365 bgp_keepalives_off(peer
);
366 BGP_TIMER_OFF(peer
->t_routeadv
);
370 /* After start timer is expired, the peer moves to Connect
371 status. Make sure start timer is off and connect timer is
373 BGP_TIMER_OFF(peer
->t_start
);
374 BGP_TIMER_ON(peer
->t_connect
, bgp_connect_timer
,
376 BGP_TIMER_OFF(peer
->t_holdtime
);
377 bgp_keepalives_off(peer
);
378 BGP_TIMER_OFF(peer
->t_routeadv
);
382 /* Active is waiting connection from remote peer. And if
383 connect timer is expired, change status to Connect. */
384 BGP_TIMER_OFF(peer
->t_start
);
385 /* If peer is passive mode, do not set connect timer. */
386 if (CHECK_FLAG(peer
->flags
, PEER_FLAG_PASSIVE
)
387 || CHECK_FLAG(peer
->sflags
, PEER_STATUS_NSF_WAIT
)) {
388 BGP_TIMER_OFF(peer
->t_connect
);
390 BGP_TIMER_ON(peer
->t_connect
, bgp_connect_timer
,
393 BGP_TIMER_OFF(peer
->t_holdtime
);
394 bgp_keepalives_off(peer
);
395 BGP_TIMER_OFF(peer
->t_routeadv
);
399 /* OpenSent status. */
400 BGP_TIMER_OFF(peer
->t_start
);
401 BGP_TIMER_OFF(peer
->t_connect
);
402 if (peer
->v_holdtime
!= 0) {
403 BGP_TIMER_ON(peer
->t_holdtime
, bgp_holdtime_timer
,
406 BGP_TIMER_OFF(peer
->t_holdtime
);
408 bgp_keepalives_off(peer
);
409 BGP_TIMER_OFF(peer
->t_routeadv
);
413 /* OpenConfirm status. */
414 BGP_TIMER_OFF(peer
->t_start
);
415 BGP_TIMER_OFF(peer
->t_connect
);
417 /* If the negotiated Hold Time value is zero, then the Hold Time
418 timer and KeepAlive timers are not started. */
419 if (peer
->v_holdtime
== 0) {
420 BGP_TIMER_OFF(peer
->t_holdtime
);
421 bgp_keepalives_off(peer
);
423 BGP_TIMER_ON(peer
->t_holdtime
, bgp_holdtime_timer
,
425 bgp_keepalives_on(peer
);
427 BGP_TIMER_OFF(peer
->t_routeadv
);
431 /* In Established status start and connect timer is turned
433 BGP_TIMER_OFF(peer
->t_start
);
434 BGP_TIMER_OFF(peer
->t_connect
);
436 /* Same as OpenConfirm, if holdtime is zero then both holdtime
437 and keepalive must be turned off. */
438 if (peer
->v_holdtime
== 0) {
439 BGP_TIMER_OFF(peer
->t_holdtime
);
440 bgp_keepalives_off(peer
);
442 BGP_TIMER_ON(peer
->t_holdtime
, bgp_holdtime_timer
,
444 bgp_keepalives_on(peer
);
448 BGP_TIMER_OFF(peer
->t_gr_restart
);
449 BGP_TIMER_OFF(peer
->t_gr_stale
);
450 BGP_TIMER_OFF(peer
->t_pmax_restart
);
453 BGP_TIMER_OFF(peer
->t_start
);
454 BGP_TIMER_OFF(peer
->t_connect
);
455 BGP_TIMER_OFF(peer
->t_holdtime
);
456 bgp_keepalives_off(peer
);
457 BGP_TIMER_OFF(peer
->t_routeadv
);
460 flog_err(EC_LIB_DEVELOPMENT
,
461 "BGP_STATUS_MAX while a legal state is not valid state for the FSM");
466 /* BGP start timer. This function set BGP_Start event to thread value
467 and process event. */
468 static int bgp_start_timer(struct thread
*thread
)
472 peer
= THREAD_ARG(thread
);
473 peer
->t_start
= NULL
;
475 if (bgp_debug_neighbor_events(peer
))
476 zlog_debug("%s [FSM] Timer (start timer expire).", peer
->host
);
478 THREAD_VAL(thread
) = BGP_Start
;
479 bgp_event(thread
); /* bgp_event unlocks peer */
484 /* BGP connect retry timer. */
485 static int bgp_connect_timer(struct thread
*thread
)
490 peer
= THREAD_ARG(thread
);
492 assert(!peer
->t_write
);
493 assert(!peer
->t_read
);
495 peer
->t_connect
= NULL
;
497 if (bgp_debug_neighbor_events(peer
))
498 zlog_debug("%s [FSM] Timer (connect timer expire)", peer
->host
);
500 if (CHECK_FLAG(peer
->sflags
, PEER_STATUS_ACCEPT_PEER
)) {
504 THREAD_VAL(thread
) = ConnectRetry_timer_expired
;
505 bgp_event(thread
); /* bgp_event unlocks peer */
512 /* BGP holdtime timer. */
513 static int bgp_holdtime_timer(struct thread
*thread
)
515 atomic_size_t inq_count
;
518 peer
= THREAD_ARG(thread
);
519 peer
->t_holdtime
= NULL
;
521 if (bgp_debug_neighbor_events(peer
))
522 zlog_debug("%s [FSM] Timer (holdtime timer expire)",
526 * Given that we do not have any expectation of ordering
527 * for handling packets from a peer -vs- handling
528 * the hold timer for a peer as that they are both
529 * events on the peer. If we have incoming
530 * data on the peers inq, let's give the system a chance
531 * to handle that data. This can be especially true
532 * for systems where we are heavily loaded for one
535 inq_count
= atomic_load_explicit(&peer
->ibuf
->count
,
536 memory_order_relaxed
);
538 BGP_TIMER_ON(peer
->t_holdtime
, bgp_holdtime_timer
,
544 THREAD_VAL(thread
) = Hold_Timer_expired
;
545 bgp_event(thread
); /* bgp_event unlocks peer */
550 int bgp_routeadv_timer(struct thread
*thread
)
554 peer
= THREAD_ARG(thread
);
555 peer
->t_routeadv
= NULL
;
557 if (bgp_debug_neighbor_events(peer
))
558 zlog_debug("%s [FSM] Timer (routeadv timer expire)",
561 peer
->synctime
= bgp_clock();
563 thread_add_timer_msec(bm
->master
, bgp_generate_updgrp_packets
, peer
, 0,
564 &peer
->t_generate_updgrp_packets
);
566 /* MRAI timer will be started again when FIFO is built, no need to
572 /* BGP Peer Down Cause */
573 const char *const peer_down_str
[] = {"",
577 "Cluster ID changed",
578 "Confederation identifier changed",
579 "Confederation peer changed",
580 "RR client config change",
581 "RS client config change",
582 "Update source change",
583 "Address family activated",
586 "BGP Notification received",
587 "BGP Notification send",
588 "Peer closed the session",
590 "Peer-group add member",
591 "Peer-group delete member",
592 "Capability changed",
593 "Passive config change",
594 "Multihop config change",
595 "NSF peer closed the session",
596 "Intf peering v6only config change",
599 "Neighbor address lost",
601 "Waiting for Peer IPv6 LLA",
602 "Waiting for VRF to be initialized",
603 "No AFI/SAFI activated for peer",
604 "AS Set config change",
605 "Waiting for peer OPEN",
606 "Reached received prefix count"};
608 static int bgp_graceful_restart_timer_expire(struct thread
*thread
)
614 peer
= THREAD_ARG(thread
);
615 peer
->t_gr_restart
= NULL
;
617 /* NSF delete stale route */
618 for (afi
= AFI_IP
; afi
< AFI_MAX
; afi
++)
619 for (safi
= SAFI_UNICAST
; safi
<= SAFI_MPLS_VPN
; safi
++)
620 if (peer
->nsf
[afi
][safi
])
621 bgp_clear_stale_route(peer
, afi
, safi
);
623 UNSET_FLAG(peer
->sflags
, PEER_STATUS_NSF_WAIT
);
624 BGP_TIMER_OFF(peer
->t_gr_stale
);
626 if (bgp_debug_neighbor_events(peer
)) {
627 zlog_debug("%s graceful restart timer expired", peer
->host
);
628 zlog_debug("%s graceful restart stalepath timer stopped",
637 static int bgp_graceful_stale_timer_expire(struct thread
*thread
)
643 peer
= THREAD_ARG(thread
);
644 peer
->t_gr_stale
= NULL
;
646 if (bgp_debug_neighbor_events(peer
))
647 zlog_debug("%s graceful restart stalepath timer expired",
650 /* NSF delete stale route */
651 for (afi
= AFI_IP
; afi
< AFI_MAX
; afi
++)
652 for (safi
= SAFI_UNICAST
; safi
<= SAFI_MPLS_VPN
; safi
++)
653 if (peer
->nsf
[afi
][safi
])
654 bgp_clear_stale_route(peer
, afi
, safi
);
659 /* Selection deferral timer processing function */
660 static int bgp_graceful_deferral_timer_expire(struct thread
*thread
)
662 struct afi_safi_info
*info
;
667 info
= THREAD_ARG(thread
);
672 if (BGP_DEBUG(update
, UPDATE_OUT
))
674 "afi %d, safi %d : graceful restart deferral timer expired",
677 bgp
->gr_info
[afi
][safi
].t_select_deferral
= NULL
;
679 bgp
->gr_info
[afi
][safi
].eor_required
= 0;
680 bgp
->gr_info
[afi
][safi
].eor_received
= 0;
681 XFREE(MTYPE_TMP
, info
);
683 /* Best path selection */
684 return bgp_best_path_select_defer(bgp
, afi
, safi
);
687 static bool bgp_update_delay_applicable(struct bgp
*bgp
)
689 /* update_delay_over flag should be reset (set to 0) for any new
690 applicability of the update-delay during BGP process lifetime.
691 And it should be set after an occurence of the update-delay is
693 if (!bgp
->update_delay_over
)
698 bool bgp_update_delay_active(struct bgp
*bgp
)
700 if (bgp
->t_update_delay
)
705 bool bgp_update_delay_configured(struct bgp
*bgp
)
707 if (bgp
->v_update_delay
)
712 /* Do the post-processing needed when bgp comes out of the read-only mode
713 on ending the update delay. */
714 void bgp_update_delay_end(struct bgp
*bgp
)
716 THREAD_TIMER_OFF(bgp
->t_update_delay
);
717 THREAD_TIMER_OFF(bgp
->t_establish_wait
);
719 /* Reset update-delay related state */
720 bgp
->update_delay_over
= 1;
721 bgp
->established
= 0;
722 bgp
->restarted_peers
= 0;
723 bgp
->implicit_eors
= 0;
724 bgp
->explicit_eors
= 0;
726 quagga_timestamp(3, bgp
->update_delay_end_time
,
727 sizeof(bgp
->update_delay_end_time
));
730 * Add an end-of-initial-update marker to the main process queues so
732 * the route advertisement timer for the peers can be started. Also set
733 * the zebra and peer update hold flags. These flags are used to achieve
734 * three stages in the update-delay post processing:
735 * 1. Finish best-path selection for all the prefixes held on the
737 * (routes in BGP are updated, and peers sync queues are populated
739 * 2. As the eoiu mark is reached in the bgp process routine, ship all
741 * routes to zebra. With that zebra should see updates from BGP
744 * 3. Unblock the peer update writes. With that peer update packing
746 * the prefixes should be at its maximum.
748 bgp_add_eoiu_mark(bgp
);
749 bgp
->main_zebra_update_hold
= 1;
750 bgp
->main_peers_update_hold
= 1;
752 /* Resume the queue processing. This should trigger the event that would
754 care of processing any work that was queued during the read-only
756 work_queue_unplug(bm
->process_main_queue
);
762 void bgp_start_routeadv(struct bgp
*bgp
)
764 struct listnode
*node
, *nnode
;
767 zlog_info("bgp_start_routeadv(), update hold status %d",
768 bgp
->main_peers_update_hold
);
770 if (bgp
->main_peers_update_hold
)
773 quagga_timestamp(3, bgp
->update_delay_peers_resume_time
,
774 sizeof(bgp
->update_delay_peers_resume_time
));
776 for (ALL_LIST_ELEMENTS(bgp
->peer
, node
, nnode
, peer
)) {
777 if (peer
->status
!= Established
)
779 BGP_TIMER_OFF(peer
->t_routeadv
);
780 BGP_TIMER_ON(peer
->t_routeadv
, bgp_routeadv_timer
, 0);
787 void bgp_adjust_routeadv(struct peer
*peer
)
789 time_t nowtime
= bgp_clock();
791 unsigned long remain
;
793 /* Bypass checks for special case of MRAI being 0 */
794 if (peer
->v_routeadv
== 0) {
795 /* Stop existing timer, just in case it is running for a
797 * duration and schedule write thread immediately.
799 if (peer
->t_routeadv
)
800 BGP_TIMER_OFF(peer
->t_routeadv
);
802 peer
->synctime
= bgp_clock();
803 thread_add_timer_msec(bm
->master
, bgp_generate_updgrp_packets
,
805 &peer
->t_generate_updgrp_packets
);
812 * If the last update was written more than MRAI back, expire the timer
813 * instantly so that we can send the update out sooner.
815 * <------- MRAI --------->
816 * |-----------------|-----------------------|
817 * <------------- m ------------>
826 diff
= difftime(nowtime
, peer
->last_update
);
827 if (diff
> (double)peer
->v_routeadv
) {
828 BGP_TIMER_OFF(peer
->t_routeadv
);
829 BGP_TIMER_ON(peer
->t_routeadv
, bgp_routeadv_timer
, 0);
835 * - Find when to expire the MRAI timer.
836 * If MRAI timer is not active, assume we can start it now.
838 * <------- MRAI --------->
839 * |------------|-----------------------|
840 * <-------- m ----------><----- r ----->
849 if (peer
->t_routeadv
)
850 remain
= thread_timer_remain_second(peer
->t_routeadv
);
852 remain
= peer
->v_routeadv
;
853 diff
= peer
->v_routeadv
- diff
;
854 if (diff
<= (double)remain
) {
855 BGP_TIMER_OFF(peer
->t_routeadv
);
856 BGP_TIMER_ON(peer
->t_routeadv
, bgp_routeadv_timer
, diff
);
860 static bool bgp_maxmed_onstartup_applicable(struct bgp
*bgp
)
862 if (!bgp
->maxmed_onstartup_over
)
867 bool bgp_maxmed_onstartup_configured(struct bgp
*bgp
)
869 if (bgp
->v_maxmed_onstartup
!= BGP_MAXMED_ONSTARTUP_UNCONFIGURED
)
874 bool bgp_maxmed_onstartup_active(struct bgp
*bgp
)
876 if (bgp
->t_maxmed_onstartup
)
881 void bgp_maxmed_update(struct bgp
*bgp
)
883 uint8_t maxmed_active
;
884 uint32_t maxmed_value
;
886 if (bgp
->v_maxmed_admin
) {
888 maxmed_value
= bgp
->maxmed_admin_value
;
889 } else if (bgp
->t_maxmed_onstartup
) {
891 maxmed_value
= bgp
->maxmed_onstartup_value
;
894 maxmed_value
= BGP_MAXMED_VALUE_DEFAULT
;
897 if (bgp
->maxmed_active
!= maxmed_active
898 || bgp
->maxmed_value
!= maxmed_value
) {
899 bgp
->maxmed_active
= maxmed_active
;
900 bgp
->maxmed_value
= maxmed_value
;
902 update_group_announce(bgp
);
906 int bgp_fsm_error_subcode(int status
)
908 int fsm_err_subcode
= BGP_NOTIFY_FSM_ERR_SUBCODE_UNSPECIFIC
;
912 fsm_err_subcode
= BGP_NOTIFY_FSM_ERR_SUBCODE_OPENSENT
;
915 fsm_err_subcode
= BGP_NOTIFY_FSM_ERR_SUBCODE_OPENCONFIRM
;
918 fsm_err_subcode
= BGP_NOTIFY_FSM_ERR_SUBCODE_ESTABLISHED
;
924 return fsm_err_subcode
;
927 /* The maxmed onstartup timer expiry callback. */
928 static int bgp_maxmed_onstartup_timer(struct thread
*thread
)
932 zlog_info("Max med on startup ended - timer expired.");
934 bgp
= THREAD_ARG(thread
);
935 THREAD_TIMER_OFF(bgp
->t_maxmed_onstartup
);
936 bgp
->maxmed_onstartup_over
= 1;
938 bgp_maxmed_update(bgp
);
943 static void bgp_maxmed_onstartup_begin(struct bgp
*bgp
)
945 /* Applicable only once in the process lifetime on the startup */
946 if (bgp
->maxmed_onstartup_over
)
949 zlog_info("Begin maxmed onstartup mode - timer %d seconds",
950 bgp
->v_maxmed_onstartup
);
952 thread_add_timer(bm
->master
, bgp_maxmed_onstartup_timer
, bgp
,
953 bgp
->v_maxmed_onstartup
, &bgp
->t_maxmed_onstartup
);
955 if (!bgp
->v_maxmed_admin
) {
956 bgp
->maxmed_active
= 1;
957 bgp
->maxmed_value
= bgp
->maxmed_onstartup_value
;
960 /* Route announce to all peers should happen after this in
964 static void bgp_maxmed_onstartup_process_status_change(struct peer
*peer
)
966 if (peer
->status
== Established
&& !peer
->bgp
->established
) {
967 bgp_maxmed_onstartup_begin(peer
->bgp
);
971 /* The update delay timer expiry callback. */
972 static int bgp_update_delay_timer(struct thread
*thread
)
976 zlog_info("Update delay ended - timer expired.");
978 bgp
= THREAD_ARG(thread
);
979 THREAD_TIMER_OFF(bgp
->t_update_delay
);
980 bgp_update_delay_end(bgp
);
985 /* The establish wait timer expiry callback. */
986 static int bgp_establish_wait_timer(struct thread
*thread
)
990 zlog_info("Establish wait - timer expired.");
992 bgp
= THREAD_ARG(thread
);
993 THREAD_TIMER_OFF(bgp
->t_establish_wait
);
994 bgp_check_update_delay(bgp
);
999 /* Steps to begin the update delay:
1000 - initialize queues if needed
1001 - stop the queue processing
1002 - start the timer */
1003 static void bgp_update_delay_begin(struct bgp
*bgp
)
1005 struct listnode
*node
, *nnode
;
1008 /* Stop the processing of queued work. Enqueue shall continue */
1009 work_queue_plug(bm
->process_main_queue
);
1011 for (ALL_LIST_ELEMENTS(bgp
->peer
, node
, nnode
, peer
))
1012 peer
->update_delay_over
= 0;
1014 /* Start the update-delay timer */
1015 thread_add_timer(bm
->master
, bgp_update_delay_timer
, bgp
,
1016 bgp
->v_update_delay
, &bgp
->t_update_delay
);
1018 if (bgp
->v_establish_wait
!= bgp
->v_update_delay
)
1019 thread_add_timer(bm
->master
, bgp_establish_wait_timer
, bgp
,
1020 bgp
->v_establish_wait
, &bgp
->t_establish_wait
);
1022 quagga_timestamp(3, bgp
->update_delay_begin_time
,
1023 sizeof(bgp
->update_delay_begin_time
));
1026 static void bgp_update_delay_process_status_change(struct peer
*peer
)
1028 if (peer
->status
== Established
) {
1029 if (!peer
->bgp
->established
++) {
1030 bgp_update_delay_begin(peer
->bgp
);
1032 "Begin read-only mode - update-delay timer %d seconds",
1033 peer
->bgp
->v_update_delay
);
1035 if (CHECK_FLAG(peer
->cap
, PEER_CAP_RESTART_BIT_RCV
))
1036 bgp_update_restarted_peers(peer
);
1038 if (peer
->ostatus
== Established
1039 && bgp_update_delay_active(peer
->bgp
)) {
1040 /* Adjust the update-delay state to account for this flap.
1041 NOTE: Intentionally skipping adjusting implicit_eors or
1043 counters. Extra sanity check in bgp_check_update_delay()
1045 be enough to take care of any additive discrepancy in bgp eor
1047 peer
->bgp
->established
--;
1048 peer
->update_delay_over
= 0;
1052 /* Called after event occurred, this function change status and reset
1053 read/write and timer thread. */
1054 void bgp_fsm_change_status(struct peer
*peer
, int status
)
1057 uint32_t peer_count
;
1060 peer_count
= bgp
->established_peers
;
1062 if (status
== Established
)
1063 bgp
->established_peers
++;
1064 else if ((peer
->status
== Established
) && (status
!= Established
))
1065 bgp
->established_peers
--;
1067 if (bgp_debug_neighbor_events(peer
)) {
1068 struct vrf
*vrf
= vrf_lookup_by_id(bgp
->vrf_id
);
1070 zlog_debug("%s : vrf %s(%u), Status: %s established_peers %u", __func__
,
1071 vrf
? vrf
->name
: "Unknown", bgp
->vrf_id
,
1072 lookup_msg(bgp_status_msg
, status
, NULL
),
1073 bgp
->established_peers
);
1076 /* Set to router ID to the value provided by RIB if there are no peers
1077 * in the established state and peer count did not change
1079 if ((peer_count
!= bgp
->established_peers
) &&
1080 (bgp
->established_peers
== 0))
1081 bgp_router_id_zebra_bump(bgp
->vrf_id
, NULL
);
1083 /* Transition into Clearing or Deleted must /always/ clear all routes..
1084 * (and must do so before actually changing into Deleted..
1086 if (status
>= Clearing
) {
1087 bgp_clear_route_all(peer
);
1089 /* If no route was queued for the clear-node processing,
1091 * completion event here. This is needed because if there are no
1093 * to trigger the background clear-node thread, the event won't
1095 * generated and the peer would be stuck in Clearing. Note that
1097 * event is for the peer and helps the peer transition out of
1099 * state; it should not be generated per (AFI,SAFI). The event
1101 * directly posted here without calling clear_node_complete() as
1103 * shouldn't do an extra unlock. This event will get processed
1105 * the state change that happens below, so peer will be in
1109 if (!work_queue_is_scheduled(peer
->clear_node_queue
))
1110 BGP_EVENT_ADD(peer
, Clearing_Completed
);
1113 /* Preserve old status and change into new status. */
1114 peer
->ostatus
= peer
->status
;
1115 peer
->status
= status
;
1117 /* Save event that caused status change. */
1118 peer
->last_major_event
= peer
->cur_event
;
1120 /* Operations after status change */
1121 hook_call(peer_status_changed
, peer
);
1123 if (status
== Established
)
1124 UNSET_FLAG(peer
->sflags
, PEER_STATUS_ACCEPT_PEER
);
1126 /* If max-med processing is applicable, do the necessary. */
1127 if (status
== Established
) {
1128 if (bgp_maxmed_onstartup_configured(peer
->bgp
)
1129 && bgp_maxmed_onstartup_applicable(peer
->bgp
))
1130 bgp_maxmed_onstartup_process_status_change(peer
);
1132 peer
->bgp
->maxmed_onstartup_over
= 1;
1135 /* If update-delay processing is applicable, do the necessary. */
1136 if (bgp_update_delay_configured(peer
->bgp
)
1137 && bgp_update_delay_applicable(peer
->bgp
))
1138 bgp_update_delay_process_status_change(peer
);
1140 if (bgp_debug_neighbor_events(peer
))
1141 zlog_debug("%s went from %s to %s", peer
->host
,
1142 lookup_msg(bgp_status_msg
, peer
->ostatus
, NULL
),
1143 lookup_msg(bgp_status_msg
, peer
->status
, NULL
));
1146 /* Flush the event queue and ensure the peer is shut down */
1147 static int bgp_clearing_completed(struct peer
*peer
)
1149 int rc
= bgp_stop(peer
);
1152 BGP_EVENT_FLUSH(peer
);
1157 /* Administrative BGP peer stop event. */
1158 /* May be called multiple times for the same peer */
1159 int bgp_stop(struct peer
*peer
)
1163 char orf_name
[BUFSIZ
];
1165 struct bgp
*bgp
= peer
->bgp
;
1166 struct graceful_restart_info
*gr_info
= NULL
;
1168 peer
->nsf_af_count
= 0;
1170 /* deregister peer */
1171 if (peer
->last_reset
== PEER_DOWN_UPDATE_SOURCE_CHANGE
)
1172 bgp_bfd_deregister_peer(peer
);
1174 if (peer_dynamic_neighbor(peer
)
1175 && !(CHECK_FLAG(peer
->flags
, PEER_FLAG_DELETE
))) {
1176 if (bgp_debug_neighbor_events(peer
))
1177 zlog_debug("%s (dynamic neighbor) deleted", peer
->host
);
1182 /* Can't do this in Clearing; events are used for state transitions */
1183 if (peer
->status
!= Clearing
) {
1184 /* Delete all existing events of the peer */
1185 BGP_EVENT_FLUSH(peer
);
1188 /* Increment Dropped count. */
1189 if (peer
->status
== Established
) {
1192 /* bgp log-neighbor-changes of neighbor Down */
1193 if (CHECK_FLAG(peer
->bgp
->flags
,
1194 BGP_FLAG_LOG_NEIGHBOR_CHANGES
)) {
1195 struct vrf
*vrf
= vrf_lookup_by_id(peer
->bgp
->vrf_id
);
1198 "%%ADJCHANGE: neighbor %s(%s) in vrf %s Down %s",
1200 (peer
->hostname
) ? peer
->hostname
: "Unknown",
1201 vrf
? ((vrf
->vrf_id
!= VRF_DEFAULT
)
1205 peer_down_str
[(int)peer
->last_reset
]);
1208 /* graceful restart */
1209 if (peer
->t_gr_stale
) {
1210 BGP_TIMER_OFF(peer
->t_gr_stale
);
1211 if (bgp_debug_neighbor_events(peer
))
1213 "%s graceful restart stalepath timer stopped",
1216 if (CHECK_FLAG(peer
->sflags
, PEER_STATUS_NSF_WAIT
)) {
1217 if (bgp_debug_neighbor_events(peer
)) {
1219 "%s graceful restart timer started for %d sec",
1220 peer
->host
, peer
->v_gr_restart
);
1222 "%s graceful restart stalepath timer started for %d sec",
1223 peer
->host
, peer
->bgp
->stalepath_time
);
1225 BGP_TIMER_ON(peer
->t_gr_restart
,
1226 bgp_graceful_restart_timer_expire
,
1227 peer
->v_gr_restart
);
1228 BGP_TIMER_ON(peer
->t_gr_stale
,
1229 bgp_graceful_stale_timer_expire
,
1230 peer
->bgp
->stalepath_time
);
1232 UNSET_FLAG(peer
->sflags
, PEER_STATUS_NSF_MODE
);
1234 for (afi
= AFI_IP
; afi
< AFI_MAX
; afi
++)
1235 for (safi
= SAFI_UNICAST
; safi
<= SAFI_MPLS_VPN
;
1237 peer
->nsf
[afi
][safi
] = 0;
1240 /* If peer reset before receiving EOR, decrement EOR count and
1241 * cancel the selection deferral timer if there are no
1242 * pending EOR messages to be received
1244 if (BGP_PEER_GRACEFUL_RESTART_CAPABLE(peer
)) {
1245 FOREACH_AFI_SAFI (afi
, safi
) {
1246 if (!peer
->afc_nego
[afi
][safi
]
1247 || CHECK_FLAG(peer
->af_sflags
[afi
][safi
],
1248 PEER_STATUS_EOR_RECEIVED
))
1251 gr_info
= &bgp
->gr_info
[afi
][safi
];
1255 if (gr_info
->eor_required
)
1256 gr_info
->eor_required
--;
1258 if (BGP_DEBUG(update
, UPDATE_OUT
))
1259 zlog_debug("peer %s, EOR_required %d",
1261 gr_info
->eor_required
);
1263 /* There is no pending EOR message */
1264 if (gr_info
->eor_required
== 0) {
1266 gr_info
->t_select_deferral
);
1267 gr_info
->eor_received
= 0;
1272 /* set last reset time */
1273 peer
->resettime
= peer
->uptime
= bgp_clock();
1275 if (BGP_DEBUG(update_groups
, UPDATE_GROUPS
))
1276 zlog_debug("%s remove from all update group",
1278 update_group_remove_peer_afs(peer
);
1280 hook_call(peer_backward_transition
, peer
);
1282 /* Reset peer synctime */
1286 /* stop keepalives */
1287 bgp_keepalives_off(peer
);
1289 /* Stop read and write threads. */
1290 bgp_writes_off(peer
);
1291 bgp_reads_off(peer
);
1293 THREAD_OFF(peer
->t_connect_check_r
);
1294 THREAD_OFF(peer
->t_connect_check_w
);
1296 /* Stop all timers. */
1297 BGP_TIMER_OFF(peer
->t_start
);
1298 BGP_TIMER_OFF(peer
->t_connect
);
1299 BGP_TIMER_OFF(peer
->t_holdtime
);
1300 BGP_TIMER_OFF(peer
->t_routeadv
);
1302 /* Clear input and output buffer. */
1303 frr_with_mutex(&peer
->io_mtx
) {
1305 stream_fifo_clean(peer
->ibuf
);
1307 stream_fifo_clean(peer
->obuf
);
1309 if (peer
->ibuf_work
)
1310 ringbuf_wipe(peer
->ibuf_work
);
1311 if (peer
->obuf_work
)
1312 stream_reset(peer
->obuf_work
);
1315 stream_free(peer
->curr
);
1320 /* Close of file descriptor. */
1321 if (peer
->fd
>= 0) {
1326 FOREACH_AFI_SAFI (afi
, safi
) {
1327 /* Reset all negotiated variables */
1328 peer
->afc_nego
[afi
][safi
] = 0;
1329 peer
->afc_adv
[afi
][safi
] = 0;
1330 peer
->afc_recv
[afi
][safi
] = 0;
1332 /* peer address family capability flags*/
1333 peer
->af_cap
[afi
][safi
] = 0;
1335 /* peer address family status flags*/
1336 peer
->af_sflags
[afi
][safi
] = 0;
1338 /* Received ORF prefix-filter */
1339 peer
->orf_plist
[afi
][safi
] = NULL
;
1341 if ((peer
->status
== OpenConfirm
)
1342 || (peer
->status
== Established
)) {
1343 /* ORF received prefix-filter pnt */
1344 snprintf(orf_name
, sizeof(orf_name
), "%s.%d.%d",
1345 peer
->host
, afi
, safi
);
1346 prefix_bgp_orf_remove_all(afi
, orf_name
);
1350 /* Reset keepalive and holdtime */
1351 if (CHECK_FLAG(peer
->flags
, PEER_FLAG_TIMER
)) {
1352 peer
->v_keepalive
= peer
->keepalive
;
1353 peer
->v_holdtime
= peer
->holdtime
;
1355 peer
->v_keepalive
= peer
->bgp
->default_keepalive
;
1356 peer
->v_holdtime
= peer
->bgp
->default_holdtime
;
1359 peer
->update_time
= 0;
1361 /* Until we are sure that there is no problem about prefix count
1362 this should be commented out.*/
1364 /* Reset prefix count */
1365 peer
->pcount
[AFI_IP
][SAFI_UNICAST
] = 0;
1366 peer
->pcount
[AFI_IP
][SAFI_MULTICAST
] = 0;
1367 peer
->pcount
[AFI_IP
][SAFI_LABELED_UNICAST
] = 0;
1368 peer
->pcount
[AFI_IP
][SAFI_MPLS_VPN
] = 0;
1369 peer
->pcount
[AFI_IP6
][SAFI_UNICAST
] = 0;
1370 peer
->pcount
[AFI_IP6
][SAFI_MULTICAST
] = 0;
1371 peer
->pcount
[AFI_IP6
][SAFI_LABELED_UNICAST
] = 0;
1374 if (!CHECK_FLAG(peer
->flags
, PEER_FLAG_CONFIG_NODE
)
1375 && !(CHECK_FLAG(peer
->flags
, PEER_FLAG_DELETE
))) {
1379 bgp_peer_conf_if_to_su_update(peer
);
1384 /* BGP peer is stoped by the error. */
1385 static int bgp_stop_with_error(struct peer
*peer
)
1387 /* Double start timer. */
1390 /* Overflow check. */
1391 if (peer
->v_start
>= (60 * 2))
1392 peer
->v_start
= (60 * 2);
1394 if (peer_dynamic_neighbor(peer
)) {
1395 if (bgp_debug_neighbor_events(peer
))
1396 zlog_debug("%s (dynamic neighbor) deleted", peer
->host
);
1401 return (bgp_stop(peer
));
1405 /* something went wrong, send notify and tear down */
1406 static int bgp_stop_with_notify(struct peer
*peer
, uint8_t code
,
1409 /* Send notify to remote peer */
1410 bgp_notify_send(peer
, code
, sub_code
);
1412 if (peer_dynamic_neighbor(peer
)) {
1413 if (bgp_debug_neighbor_events(peer
))
1414 zlog_debug("%s (dynamic neighbor) deleted", peer
->host
);
1419 /* Clear start timer value to default. */
1420 peer
->v_start
= BGP_INIT_START_TIMER
;
1422 return (bgp_stop(peer
));
1426 * Determines whether a TCP session has successfully established for a peer and
1427 * events as appropriate.
1429 * This function is called when setting up a new session. After connect() is
1430 * called on the peer's socket (in bgp_start()), the fd is passed to poll()
1431 * to wait for connection success or failure. When poll() returns, this
1432 * function is called to evaluate the result.
1434 * Due to differences in behavior of poll() on Linux and BSD - specifically,
1435 * the value of .revents in the case of a closed connection - this function is
1436 * scheduled both for a read and a write event. The write event is triggered
1437 * when the connection is established. A read event is triggered when the
1438 * connection is closed. Thus we need to cancel whichever one did not occur.
1440 static int bgp_connect_check(struct thread
*thread
)
1447 peer
= THREAD_ARG(thread
);
1448 assert(!CHECK_FLAG(peer
->thread_flags
, PEER_THREAD_READS_ON
));
1449 assert(!CHECK_FLAG(peer
->thread_flags
, PEER_THREAD_WRITES_ON
));
1450 assert(!peer
->t_read
);
1451 assert(!peer
->t_write
);
1453 THREAD_OFF(peer
->t_connect_check_r
);
1454 THREAD_OFF(peer
->t_connect_check_w
);
1456 /* Check file descriptor. */
1457 slen
= sizeof(status
);
1458 ret
= getsockopt(peer
->fd
, SOL_SOCKET
, SO_ERROR
, (void *)&status
,
1461 /* If getsockopt is fail, this is fatal error. */
1463 zlog_err("can't get sockopt for nonblocking connect: %d(%s)",
1464 errno
, safe_strerror(errno
));
1465 BGP_EVENT_ADD(peer
, TCP_fatal_error
);
1469 /* When status is 0 then TCP connection is established. */
1471 BGP_EVENT_ADD(peer
, TCP_connection_open
);
1474 if (bgp_debug_neighbor_events(peer
))
1475 zlog_debug("%s [Event] Connect failed %d(%s)",
1476 peer
->host
, status
, safe_strerror(status
));
1477 BGP_EVENT_ADD(peer
, TCP_connection_open_failed
);
1482 /* TCP connection open. Next we send open message to remote peer. And
1483 add read thread for reading open message. */
1484 static int bgp_connect_success(struct peer
*peer
)
1487 flog_err(EC_BGP_CONNECT
,
1488 "bgp_connect_success peer's fd is negative value %d",
1494 if (bgp_getsockname(peer
) < 0) {
1495 flog_err_sys(EC_LIB_SOCKET
,
1496 "%s: bgp_getsockname(): failed for peer %s, fd %d",
1497 __func__
, peer
->host
, peer
->fd
);
1498 bgp_notify_send(peer
, BGP_NOTIFY_FSM_ERR
,
1499 bgp_fsm_error_subcode(peer
->status
));
1500 bgp_writes_on(peer
);
1506 if (bgp_debug_neighbor_events(peer
)) {
1507 char buf1
[SU_ADDRSTRLEN
];
1509 if (!CHECK_FLAG(peer
->sflags
, PEER_STATUS_ACCEPT_PEER
))
1510 zlog_debug("%s open active, local address %s",
1512 sockunion2str(peer
->su_local
, buf1
,
1515 zlog_debug("%s passive open", peer
->host
);
1518 bgp_open_send(peer
);
1523 /* TCP connect fail */
1524 static int bgp_connect_fail(struct peer
*peer
)
1526 if (peer_dynamic_neighbor(peer
)) {
1527 if (bgp_debug_neighbor_events(peer
))
1528 zlog_debug("%s (dynamic neighbor) deleted", peer
->host
);
1533 return (bgp_stop(peer
));
1536 /* This function is the first starting point of all BGP connection. It
1537 try to connect to remote peer with non-blocking IO. */
1538 int bgp_start(struct peer
*peer
)
1542 bgp_peer_conf_if_to_su_update(peer
);
1544 if (peer
->su
.sa
.sa_family
== AF_UNSPEC
) {
1545 if (bgp_debug_neighbor_events(peer
))
1547 "%s [FSM] Unable to get neighbor's IP address, waiting...",
1549 peer
->last_reset
= PEER_DOWN_NBR_ADDR
;
1553 if (BGP_PEER_START_SUPPRESSED(peer
)) {
1554 if (bgp_debug_neighbor_events(peer
))
1555 flog_err(EC_BGP_FSM
,
1556 "%s [FSM] Trying to start suppressed peer - this is never supposed to happen!",
1558 if (CHECK_FLAG(peer
->flags
, PEER_FLAG_SHUTDOWN
))
1559 peer
->last_reset
= PEER_DOWN_USER_SHUTDOWN
;
1560 else if (CHECK_FLAG(peer
->sflags
, PEER_STATUS_PREFIX_OVERFLOW
))
1561 peer
->last_reset
= PEER_DOWN_PFX_COUNT
;
1565 /* Scrub some information that might be left over from a previous,
1568 /* Connection information. */
1569 if (peer
->su_local
) {
1570 sockunion_free(peer
->su_local
);
1571 peer
->su_local
= NULL
;
1574 if (peer
->su_remote
) {
1575 sockunion_free(peer
->su_remote
);
1576 peer
->su_remote
= NULL
;
1579 /* Clear remote router-id. */
1580 peer
->remote_id
.s_addr
= INADDR_ANY
;
1582 /* Clear peer capability flag. */
1585 /* If the peer is passive mode, force to move to Active mode. */
1586 if (CHECK_FLAG(peer
->flags
, PEER_FLAG_PASSIVE
)) {
1587 BGP_EVENT_ADD(peer
, TCP_connection_open_failed
);
1591 if (peer
->bgp
->inst_type
!= BGP_INSTANCE_TYPE_VIEW
&&
1592 peer
->bgp
->vrf_id
== VRF_UNKNOWN
) {
1593 if (bgp_debug_neighbor_events(peer
))
1596 "%s [FSM] In a VRF that is not initialised yet",
1598 peer
->last_reset
= PEER_DOWN_VRF_UNINIT
;
1602 /* Register peer for NHT. If next hop is already resolved, proceed
1603 * with connection setup, else wait.
1605 if (!bgp_peer_reg_with_nht(peer
)) {
1606 if (bgp_zebra_num_connects()) {
1607 if (bgp_debug_neighbor_events(peer
))
1608 zlog_debug("%s [FSM] Waiting for NHT",
1610 peer
->last_reset
= PEER_DOWN_WAITING_NHT
;
1611 BGP_EVENT_ADD(peer
, TCP_connection_open_failed
);
1616 assert(!peer
->t_write
);
1617 assert(!peer
->t_read
);
1618 assert(!CHECK_FLAG(peer
->thread_flags
, PEER_THREAD_WRITES_ON
));
1619 assert(!CHECK_FLAG(peer
->thread_flags
, PEER_THREAD_READS_ON
));
1620 status
= bgp_connect(peer
);
1624 if (bgp_debug_neighbor_events(peer
))
1625 zlog_debug("%s [FSM] Connect error", peer
->host
);
1626 BGP_EVENT_ADD(peer
, TCP_connection_open_failed
);
1628 case connect_success
:
1629 if (bgp_debug_neighbor_events(peer
))
1631 "%s [FSM] Connect immediately success, fd %d",
1632 peer
->host
, peer
->fd
);
1633 BGP_EVENT_ADD(peer
, TCP_connection_open
);
1635 case connect_in_progress
:
1636 /* To check nonblocking connect, we wait until socket is
1637 readable or writable. */
1638 if (bgp_debug_neighbor_events(peer
))
1640 "%s [FSM] Non blocking connect waiting result, fd %d",
1641 peer
->host
, peer
->fd
);
1643 flog_err(EC_BGP_FSM
,
1644 "bgp_start peer's fd is negative value %d",
1649 * - when the socket becomes ready, poll() will signify POLLOUT
1650 * - if it fails to connect, poll() will signify POLLHUP
1651 * - POLLHUP is handled as a 'read' event by thread.c
1653 * therefore, we schedule both a read and a write event with
1654 * bgp_connect_check() as the handler for each and cancel the
1655 * unused event in that function.
1657 thread_add_read(bm
->master
, bgp_connect_check
, peer
, peer
->fd
,
1658 &peer
->t_connect_check_r
);
1659 thread_add_write(bm
->master
, bgp_connect_check
, peer
, peer
->fd
,
1660 &peer
->t_connect_check_w
);
1666 /* Connect retry timer is expired when the peer status is Connect. */
1667 static int bgp_reconnect(struct peer
*peer
)
1669 if (bgp_stop(peer
) < 0)
1672 /* Send graceful restart capabilty */
1673 BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(peer
->bgp
,
1680 static int bgp_fsm_open(struct peer
*peer
)
1682 /* Send keepalive and make keepalive timer */
1683 bgp_keepalive_send(peer
);
1685 /* Reset holdtimer value. */
1686 BGP_TIMER_OFF(peer
->t_holdtime
);
1691 /* FSM error, unexpected event. This is error of BGP connection. So cut the
1692 peer and change to Idle status. */
1693 static int bgp_fsm_event_error(struct peer
*peer
)
1695 flog_err(EC_BGP_FSM
, "%s [FSM] unexpected packet received in state %s",
1696 peer
->host
, lookup_msg(bgp_status_msg
, peer
->status
, NULL
));
1698 return bgp_stop_with_notify(peer
, BGP_NOTIFY_FSM_ERR
,
1699 bgp_fsm_error_subcode(peer
->status
));
1702 /* Hold timer expire. This is error of BGP connection. So cut the
1703 peer and change to Idle status. */
1704 static int bgp_fsm_holdtime_expire(struct peer
*peer
)
1706 if (bgp_debug_neighbor_events(peer
))
1707 zlog_debug("%s [FSM] Hold timer expire", peer
->host
);
1709 return bgp_stop_with_notify(peer
, BGP_NOTIFY_HOLD_ERR
, 0);
1712 /* Start the selection deferral timer thread for the specified AFI, SAFI */
1713 static int bgp_start_deferral_timer(struct bgp
*bgp
, afi_t afi
, safi_t safi
,
1714 struct graceful_restart_info
*gr_info
)
1716 struct afi_safi_info
*thread_info
;
1718 /* If the deferral timer is active, then increment eor count */
1719 if (gr_info
->t_select_deferral
) {
1720 gr_info
->eor_required
++;
1724 /* Start the deferral timer when the first peer enabled for the graceful
1725 * restart is established
1727 if (gr_info
->eor_required
== 0) {
1728 thread_info
= XMALLOC(MTYPE_TMP
, sizeof(struct afi_safi_info
));
1730 thread_info
->afi
= afi
;
1731 thread_info
->safi
= safi
;
1732 thread_info
->bgp
= bgp
;
1734 thread_add_timer(bm
->master
, bgp_graceful_deferral_timer_expire
,
1735 thread_info
, bgp
->select_defer_time
,
1736 &gr_info
->t_select_deferral
);
1738 gr_info
->eor_required
++;
1739 /* Send message to RIB indicating route update pending */
1740 if (gr_info
->af_enabled
[afi
][safi
] == false) {
1741 gr_info
->af_enabled
[afi
][safi
] = true;
1742 /* Send message to RIB */
1743 bgp_zebra_update(afi
, safi
, bgp
->vrf_id
,
1744 ZEBRA_CLIENT_ROUTE_UPDATE_PENDING
);
1746 if (BGP_DEBUG(update
, UPDATE_OUT
))
1747 zlog_debug("Started the deferral timer for %s eor_required %d",
1748 get_afi_safi_str(afi
, safi
, false),
1749 gr_info
->eor_required
);
1753 /* Update the graceful restart information for the specified AFI, SAFI */
1754 static int bgp_update_gr_info(struct peer
*peer
, afi_t afi
, safi_t safi
)
1756 struct graceful_restart_info
*gr_info
;
1757 struct bgp
*bgp
= peer
->bgp
;
1760 if ((afi
< AFI_IP
) || (afi
>= AFI_MAX
)) {
1761 if (BGP_DEBUG(update
, UPDATE_OUT
))
1762 zlog_debug("%s : invalid afi %d", __func__
, afi
);
1766 if ((safi
< SAFI_UNICAST
) || (safi
> SAFI_MPLS_VPN
)) {
1767 if (BGP_DEBUG(update
, UPDATE_OUT
))
1768 zlog_debug("%s : invalid safi %d", __func__
, safi
);
1772 /* Restarting router */
1773 if (BGP_PEER_GRACEFUL_RESTART_CAPABLE(peer
)
1774 && BGP_PEER_RESTARTING_MODE(peer
)) {
1775 /* Check if the forwarding state is preserved */
1776 if (CHECK_FLAG(bgp
->flags
, BGP_FLAG_GR_PRESERVE_FWD
)) {
1777 gr_info
= &(bgp
->gr_info
[afi
][safi
]);
1778 ret
= bgp_start_deferral_timer(bgp
, afi
, safi
, gr_info
);
1785 * Transition to Established state.
1787 * Convert peer from stub to full fledged peer, set some timers, and generate
1790 static int bgp_establish(struct peer
*peer
)
1794 int nsf_af_count
= 0;
1799 other
= peer
->doppelganger
;
1800 peer
= peer_xfer_conn(peer
);
1802 flog_err(EC_BGP_CONNECT
, "%%Neighbor failed in xfer_conn");
1807 ret
= 1; /* bgp_establish specific code when xfer_conn
1810 /* Reset capability open status flag. */
1811 if (!CHECK_FLAG(peer
->sflags
, PEER_STATUS_CAPABILITY_OPEN
))
1812 SET_FLAG(peer
->sflags
, PEER_STATUS_CAPABILITY_OPEN
);
1814 /* Clear start timer value to default. */
1815 peer
->v_start
= BGP_INIT_START_TIMER
;
1817 /* Increment established count. */
1818 peer
->established
++;
1819 bgp_fsm_change_status(peer
, Established
);
1821 /* bgp log-neighbor-changes of neighbor Up */
1822 if (CHECK_FLAG(peer
->bgp
->flags
, BGP_FLAG_LOG_NEIGHBOR_CHANGES
)) {
1823 struct vrf
*vrf
= vrf_lookup_by_id(peer
->bgp
->vrf_id
);
1825 "%%ADJCHANGE: neighbor %s(%s) in vrf %s Up", peer
->host
,
1826 (peer
->hostname
) ? peer
->hostname
: "Unknown",
1827 vrf
? ((vrf
->vrf_id
!= VRF_DEFAULT
) ? vrf
->name
1831 /* assign update-group/subgroup */
1832 update_group_adjust_peer_afs(peer
);
1834 /* graceful restart */
1835 UNSET_FLAG(peer
->sflags
, PEER_STATUS_NSF_WAIT
);
1836 if (bgp_debug_neighbor_events(peer
)) {
1837 if (BGP_PEER_RESTARTING_MODE(peer
))
1838 zlog_debug("peer %s BGP_RESTARTING_MODE", peer
->host
);
1839 else if (BGP_PEER_HELPER_MODE(peer
))
1840 zlog_debug("peer %s BGP_HELPER_MODE", peer
->host
);
1842 for (afi
= AFI_IP
; afi
< AFI_MAX
; afi
++)
1843 for (safi
= SAFI_UNICAST
; safi
<= SAFI_MPLS_VPN
; safi
++) {
1844 if (peer
->afc_nego
[afi
][safi
]
1845 && CHECK_FLAG(peer
->cap
, PEER_CAP_RESTART_ADV
)
1846 && CHECK_FLAG(peer
->af_cap
[afi
][safi
],
1847 PEER_CAP_RESTART_AF_RCV
)) {
1848 if (peer
->nsf
[afi
][safi
]
1850 peer
->af_cap
[afi
][safi
],
1851 PEER_CAP_RESTART_AF_PRESERVE_RCV
))
1852 bgp_clear_stale_route(peer
, afi
, safi
);
1854 peer
->nsf
[afi
][safi
] = 1;
1857 if (peer
->nsf
[afi
][safi
])
1858 bgp_clear_stale_route(peer
, afi
, safi
);
1859 peer
->nsf
[afi
][safi
] = 0;
1861 /* Update the graceful restart information */
1862 if (peer
->afc_nego
[afi
][safi
]) {
1863 if (!BGP_SELECT_DEFER_DISABLE(peer
->bgp
)) {
1864 status
= bgp_update_gr_info(peer
, afi
,
1868 "Error in updating graceful restart for %s",
1873 if (BGP_PEER_GRACEFUL_RESTART_CAPABLE(
1875 && BGP_PEER_RESTARTING_MODE(peer
)
1878 BGP_FLAG_GR_PRESERVE_FWD
))
1879 peer
->bgp
->gr_info
[afi
][safi
]
1885 peer
->nsf_af_count
= nsf_af_count
;
1888 SET_FLAG(peer
->sflags
, PEER_STATUS_NSF_MODE
);
1890 UNSET_FLAG(peer
->sflags
, PEER_STATUS_NSF_MODE
);
1891 if (peer
->t_gr_stale
) {
1892 BGP_TIMER_OFF(peer
->t_gr_stale
);
1893 if (bgp_debug_neighbor_events(peer
))
1895 "%s graceful restart stalepath timer stopped",
1900 if (peer
->t_gr_restart
) {
1901 BGP_TIMER_OFF(peer
->t_gr_restart
);
1902 if (bgp_debug_neighbor_events(peer
))
1903 zlog_debug("%s graceful restart timer stopped",
1907 /* Reset uptime, turn on keepalives, send current table. */
1908 if (!peer
->v_holdtime
)
1909 bgp_keepalives_on(peer
);
1911 peer
->uptime
= bgp_clock();
1913 /* Send route-refresh when ORF is enabled */
1914 FOREACH_AFI_SAFI (afi
, safi
) {
1915 if (CHECK_FLAG(peer
->af_cap
[afi
][safi
],
1916 PEER_CAP_ORF_PREFIX_SM_ADV
)) {
1917 if (CHECK_FLAG(peer
->af_cap
[afi
][safi
],
1918 PEER_CAP_ORF_PREFIX_RM_RCV
))
1919 bgp_route_refresh_send(peer
, afi
, safi
,
1921 REFRESH_IMMEDIATE
, 0);
1922 else if (CHECK_FLAG(peer
->af_cap
[afi
][safi
],
1923 PEER_CAP_ORF_PREFIX_RM_OLD_RCV
))
1924 bgp_route_refresh_send(peer
, afi
, safi
,
1925 ORF_TYPE_PREFIX_OLD
,
1926 REFRESH_IMMEDIATE
, 0);
1930 /* First update is deferred until ORF or ROUTE-REFRESH is received */
1931 FOREACH_AFI_SAFI (afi
, safi
) {
1932 if (CHECK_FLAG(peer
->af_cap
[afi
][safi
],
1933 PEER_CAP_ORF_PREFIX_RM_ADV
))
1934 if (CHECK_FLAG(peer
->af_cap
[afi
][safi
],
1935 PEER_CAP_ORF_PREFIX_SM_RCV
)
1936 || CHECK_FLAG(peer
->af_cap
[afi
][safi
],
1937 PEER_CAP_ORF_PREFIX_SM_OLD_RCV
))
1938 SET_FLAG(peer
->af_sflags
[afi
][safi
],
1939 PEER_STATUS_ORF_WAIT_REFRESH
);
1942 bgp_announce_peer(peer
);
1944 /* Start the route advertisement timer to send updates to the peer - if
1946 * is not in read-only mode. If it is, the timer will be started at the
1948 * of read-only mode.
1950 if (!bgp_update_delay_active(peer
->bgp
)) {
1951 BGP_TIMER_OFF(peer
->t_routeadv
);
1952 BGP_TIMER_ON(peer
->t_routeadv
, bgp_routeadv_timer
, 0);
1955 if (peer
->doppelganger
&& (peer
->doppelganger
->status
!= Deleted
)) {
1956 if (bgp_debug_neighbor_events(peer
))
1958 "[Event] Deleting stub connection for peer %s",
1961 if (peer
->doppelganger
->status
> Active
)
1962 bgp_notify_send(peer
->doppelganger
, BGP_NOTIFY_CEASE
,
1963 BGP_NOTIFY_CEASE_COLLISION_RESOLUTION
);
1965 peer_delete(peer
->doppelganger
);
1969 * If we are replacing the old peer for a doppelganger
1970 * then switch it around in the bgp->peerhash
1971 * the doppelgangers su and this peer's su are the same
1972 * so the hash_release is the same for either.
1974 hash_release(peer
->bgp
->peerhash
, peer
);
1975 hash_get(peer
->bgp
->peerhash
, peer
, hash_alloc_intern
);
1977 bgp_bfd_reset_peer(peer
);
1981 /* Keepalive packet is received. */
1982 static int bgp_fsm_keepalive(struct peer
*peer
)
1984 BGP_TIMER_OFF(peer
->t_holdtime
);
1988 /* Update packet is received. */
1989 static int bgp_fsm_update(struct peer
*peer
)
1991 BGP_TIMER_OFF(peer
->t_holdtime
);
1995 /* This is empty event. */
1996 static int bgp_ignore(struct peer
*peer
)
2000 "%s [FSM] Ignoring event %s in state %s, prior events %s, %s, fd %d",
2001 peer
->host
, bgp_event_str
[peer
->cur_event
],
2002 lookup_msg(bgp_status_msg
, peer
->status
, NULL
),
2003 bgp_event_str
[peer
->last_event
],
2004 bgp_event_str
[peer
->last_major_event
], peer
->fd
);
2008 /* This is to handle unexpected events.. */
2009 static int bgp_fsm_exeption(struct peer
*peer
)
2013 "%s [FSM] Unexpected event %s in state %s, prior events %s, %s, fd %d",
2014 peer
->host
, bgp_event_str
[peer
->cur_event
],
2015 lookup_msg(bgp_status_msg
, peer
->status
, NULL
),
2016 bgp_event_str
[peer
->last_event
],
2017 bgp_event_str
[peer
->last_major_event
], peer
->fd
);
2018 return (bgp_stop(peer
));
2021 void bgp_fsm_event_update(struct peer
*peer
, int valid
)
2026 switch (peer
->status
) {
2029 BGP_EVENT_ADD(peer
, BGP_Start
);
2033 BGP_TIMER_OFF(peer
->t_connect
);
2034 BGP_EVENT_ADD(peer
, TCP_fatal_error
);
2039 BGP_TIMER_OFF(peer
->t_connect
);
2040 BGP_EVENT_ADD(peer
, ConnectRetry_timer_expired
);
2046 if (!valid
&& (peer
->gtsm_hops
== BGP_GTSM_HOPS_CONNECTED
))
2047 BGP_EVENT_ADD(peer
, TCP_fatal_error
);
2055 /* Finite State Machine structure */
2056 static const struct {
2057 int (*func
)(struct peer
*);
2058 enum bgp_fsm_status next_state
;
2059 } FSM
[BGP_STATUS_MAX
- 1][BGP_EVENTS_MAX
- 1] = {
2061 /* Idle state: In Idle state, all events other than BGP_Start is
2062 ignored. With BGP_Start event, finite state machine calls
2064 {bgp_start
, Connect
}, /* BGP_Start */
2065 {bgp_stop
, Idle
}, /* BGP_Stop */
2066 {bgp_stop
, Idle
}, /* TCP_connection_open */
2067 {bgp_stop
, Idle
}, /* TCP_connection_closed */
2068 {bgp_ignore
, Idle
}, /* TCP_connection_open_failed */
2069 {bgp_stop
, Idle
}, /* TCP_fatal_error */
2070 {bgp_ignore
, Idle
}, /* ConnectRetry_timer_expired */
2071 {bgp_ignore
, Idle
}, /* Hold_Timer_expired */
2072 {bgp_ignore
, Idle
}, /* KeepAlive_timer_expired */
2073 {bgp_ignore
, Idle
}, /* Receive_OPEN_message */
2074 {bgp_ignore
, Idle
}, /* Receive_KEEPALIVE_message */
2075 {bgp_ignore
, Idle
}, /* Receive_UPDATE_message */
2076 {bgp_ignore
, Idle
}, /* Receive_NOTIFICATION_message */
2077 {bgp_ignore
, Idle
}, /* Clearing_Completed */
2081 {bgp_ignore
, Connect
}, /* BGP_Start */
2082 {bgp_stop
, Idle
}, /* BGP_Stop */
2083 {bgp_connect_success
, OpenSent
}, /* TCP_connection_open */
2084 {bgp_stop
, Idle
}, /* TCP_connection_closed */
2085 {bgp_connect_fail
, Active
}, /* TCP_connection_open_failed */
2086 {bgp_connect_fail
, Idle
}, /* TCP_fatal_error */
2087 {bgp_reconnect
, Connect
}, /* ConnectRetry_timer_expired */
2088 {bgp_fsm_exeption
, Idle
}, /* Hold_Timer_expired */
2089 {bgp_fsm_exeption
, Idle
}, /* KeepAlive_timer_expired */
2090 {bgp_fsm_exeption
, Idle
}, /* Receive_OPEN_message */
2091 {bgp_fsm_exeption
, Idle
}, /* Receive_KEEPALIVE_message */
2092 {bgp_fsm_exeption
, Idle
}, /* Receive_UPDATE_message */
2093 {bgp_stop
, Idle
}, /* Receive_NOTIFICATION_message */
2094 {bgp_fsm_exeption
, Idle
}, /* Clearing_Completed */
2098 {bgp_ignore
, Active
}, /* BGP_Start */
2099 {bgp_stop
, Idle
}, /* BGP_Stop */
2100 {bgp_connect_success
, OpenSent
}, /* TCP_connection_open */
2101 {bgp_stop
, Idle
}, /* TCP_connection_closed */
2102 {bgp_ignore
, Active
}, /* TCP_connection_open_failed */
2103 {bgp_fsm_exeption
, Idle
}, /* TCP_fatal_error */
2104 {bgp_start
, Connect
}, /* ConnectRetry_timer_expired */
2105 {bgp_fsm_exeption
, Idle
}, /* Hold_Timer_expired */
2106 {bgp_fsm_exeption
, Idle
}, /* KeepAlive_timer_expired */
2107 {bgp_fsm_exeption
, Idle
}, /* Receive_OPEN_message */
2108 {bgp_fsm_exeption
, Idle
}, /* Receive_KEEPALIVE_message */
2109 {bgp_fsm_exeption
, Idle
}, /* Receive_UPDATE_message */
2110 {bgp_fsm_exeption
, Idle
}, /* Receive_NOTIFICATION_message */
2111 {bgp_fsm_exeption
, Idle
}, /* Clearing_Completed */
2115 {bgp_ignore
, OpenSent
}, /* BGP_Start */
2116 {bgp_stop
, Idle
}, /* BGP_Stop */
2117 {bgp_stop
, Active
}, /* TCP_connection_open */
2118 {bgp_stop
, Active
}, /* TCP_connection_closed */
2119 {bgp_stop
, Active
}, /* TCP_connection_open_failed */
2120 {bgp_stop
, Active
}, /* TCP_fatal_error */
2121 {bgp_fsm_exeption
, Idle
}, /* ConnectRetry_timer_expired */
2122 {bgp_fsm_holdtime_expire
, Idle
}, /* Hold_Timer_expired */
2123 {bgp_fsm_exeption
, Idle
}, /* KeepAlive_timer_expired */
2124 {bgp_fsm_open
, OpenConfirm
}, /* Receive_OPEN_message */
2125 {bgp_fsm_event_error
, Idle
}, /* Receive_KEEPALIVE_message */
2126 {bgp_fsm_event_error
, Idle
}, /* Receive_UPDATE_message */
2127 {bgp_fsm_event_error
, Idle
}, /* Receive_NOTIFICATION_message */
2128 {bgp_fsm_exeption
, Idle
}, /* Clearing_Completed */
2132 {bgp_ignore
, OpenConfirm
}, /* BGP_Start */
2133 {bgp_stop
, Idle
}, /* BGP_Stop */
2134 {bgp_stop
, Idle
}, /* TCP_connection_open */
2135 {bgp_stop
, Idle
}, /* TCP_connection_closed */
2136 {bgp_stop
, Idle
}, /* TCP_connection_open_failed */
2137 {bgp_stop
, Idle
}, /* TCP_fatal_error */
2138 {bgp_fsm_exeption
, Idle
}, /* ConnectRetry_timer_expired */
2139 {bgp_fsm_holdtime_expire
, Idle
}, /* Hold_Timer_expired */
2140 {bgp_ignore
, OpenConfirm
}, /* KeepAlive_timer_expired */
2141 {bgp_fsm_exeption
, Idle
}, /* Receive_OPEN_message */
2142 {bgp_establish
, Established
}, /* Receive_KEEPALIVE_message */
2143 {bgp_fsm_exeption
, Idle
}, /* Receive_UPDATE_message */
2144 {bgp_stop_with_error
, Idle
}, /* Receive_NOTIFICATION_message */
2145 {bgp_fsm_exeption
, Idle
}, /* Clearing_Completed */
2149 {bgp_ignore
, Established
}, /* BGP_Start */
2150 {bgp_stop
, Clearing
}, /* BGP_Stop */
2151 {bgp_stop
, Clearing
}, /* TCP_connection_open */
2152 {bgp_stop
, Clearing
}, /* TCP_connection_closed */
2153 {bgp_stop
, Clearing
}, /* TCP_connection_open_failed */
2154 {bgp_stop
, Clearing
}, /* TCP_fatal_error */
2155 {bgp_stop
, Clearing
}, /* ConnectRetry_timer_expired */
2156 {bgp_fsm_holdtime_expire
, Clearing
}, /* Hold_Timer_expired */
2157 {bgp_ignore
, Established
}, /* KeepAlive_timer_expired */
2158 {bgp_stop
, Clearing
}, /* Receive_OPEN_message */
2160 Established
}, /* Receive_KEEPALIVE_message */
2161 {bgp_fsm_update
, Established
}, /* Receive_UPDATE_message */
2162 {bgp_stop_with_error
,
2163 Clearing
}, /* Receive_NOTIFICATION_message */
2164 {bgp_fsm_exeption
, Idle
}, /* Clearing_Completed */
2168 {bgp_ignore
, Clearing
}, /* BGP_Start */
2169 {bgp_stop
, Clearing
}, /* BGP_Stop */
2170 {bgp_stop
, Clearing
}, /* TCP_connection_open */
2171 {bgp_stop
, Clearing
}, /* TCP_connection_closed */
2172 {bgp_stop
, Clearing
}, /* TCP_connection_open_failed */
2173 {bgp_stop
, Clearing
}, /* TCP_fatal_error */
2174 {bgp_stop
, Clearing
}, /* ConnectRetry_timer_expired */
2175 {bgp_stop
, Clearing
}, /* Hold_Timer_expired */
2176 {bgp_stop
, Clearing
}, /* KeepAlive_timer_expired */
2177 {bgp_stop
, Clearing
}, /* Receive_OPEN_message */
2178 {bgp_stop
, Clearing
}, /* Receive_KEEPALIVE_message */
2179 {bgp_stop
, Clearing
}, /* Receive_UPDATE_message */
2180 {bgp_stop
, Clearing
}, /* Receive_NOTIFICATION_message */
2181 {bgp_clearing_completed
, Idle
}, /* Clearing_Completed */
2185 {bgp_ignore
, Deleted
}, /* BGP_Start */
2186 {bgp_ignore
, Deleted
}, /* BGP_Stop */
2187 {bgp_ignore
, Deleted
}, /* TCP_connection_open */
2188 {bgp_ignore
, Deleted
}, /* TCP_connection_closed */
2189 {bgp_ignore
, Deleted
}, /* TCP_connection_open_failed */
2190 {bgp_ignore
, Deleted
}, /* TCP_fatal_error */
2191 {bgp_ignore
, Deleted
}, /* ConnectRetry_timer_expired */
2192 {bgp_ignore
, Deleted
}, /* Hold_Timer_expired */
2193 {bgp_ignore
, Deleted
}, /* KeepAlive_timer_expired */
2194 {bgp_ignore
, Deleted
}, /* Receive_OPEN_message */
2195 {bgp_ignore
, Deleted
}, /* Receive_KEEPALIVE_message */
2196 {bgp_ignore
, Deleted
}, /* Receive_UPDATE_message */
2197 {bgp_ignore
, Deleted
}, /* Receive_NOTIFICATION_message */
2198 {bgp_ignore
, Deleted
}, /* Clearing_Completed */
2202 /* Execute event process. */
2203 int bgp_event(struct thread
*thread
)
2205 enum bgp_fsm_events event
;
2209 peer
= THREAD_ARG(thread
);
2210 event
= THREAD_VAL(thread
);
2212 ret
= bgp_event_update(peer
, event
);
2217 int bgp_event_update(struct peer
*peer
, enum bgp_fsm_events event
)
2219 enum bgp_fsm_status next
;
2222 int passive_conn
= 0;
2225 /* default return code */
2226 ret
= FSM_PEER_NOOP
;
2228 other
= peer
->doppelganger
;
2230 (CHECK_FLAG(peer
->sflags
, PEER_STATUS_ACCEPT_PEER
)) ? 1 : 0;
2231 dyn_nbr
= peer_dynamic_neighbor(peer
);
2233 /* Logging this event. */
2234 next
= FSM
[peer
->status
- 1][event
- 1].next_state
;
2236 if (bgp_debug_neighbor_events(peer
) && peer
->status
!= next
)
2237 zlog_debug("%s [FSM] %s (%s->%s), fd %d", peer
->host
,
2238 bgp_event_str
[event
],
2239 lookup_msg(bgp_status_msg
, peer
->status
, NULL
),
2240 lookup_msg(bgp_status_msg
, next
, NULL
), peer
->fd
);
2242 peer
->last_event
= peer
->cur_event
;
2243 peer
->cur_event
= event
;
2245 /* Call function. */
2246 if (FSM
[peer
->status
- 1][event
- 1].func
)
2247 ret
= (*(FSM
[peer
->status
- 1][event
- 1].func
))(peer
);
2250 if (ret
== 1 && next
== Established
) {
2251 /* The case when doppelganger swap accurred in
2253 Update the peer pointer accordingly */
2254 ret
= FSM_PEER_TRANSFERRED
;
2258 /* If status is changed. */
2259 if (next
!= peer
->status
) {
2260 bgp_fsm_change_status(peer
, next
);
2263 * If we're going to ESTABLISHED then we executed a
2264 * peer transfer. In this case we can either return
2265 * FSM_PEER_TRANSITIONED or FSM_PEER_TRANSFERRED.
2266 * Opting for TRANSFERRED since transfer implies
2267 * session establishment.
2269 if (ret
!= FSM_PEER_TRANSFERRED
)
2270 ret
= FSM_PEER_TRANSITIONED
;
2273 /* Make sure timer is set. */
2274 bgp_timer_set(peer
);
2278 * If we got a return value of -1, that means there was an
2279 * error, restart the FSM. Since bgp_stop() was called on the
2280 * peer. only a few fields are safe to access here. In any case
2281 * we need to indicate that the peer was stopped in the return
2284 if (!dyn_nbr
&& !passive_conn
&& peer
->bgp
) {
2287 "%s [FSM] Failure handling event %s in state %s, prior events %s, %s, fd %d",
2288 peer
->host
, bgp_event_str
[peer
->cur_event
],
2289 lookup_msg(bgp_status_msg
, peer
->status
, NULL
),
2290 bgp_event_str
[peer
->last_event
],
2291 bgp_event_str
[peer
->last_major_event
],
2294 bgp_fsm_change_status(peer
, Idle
);
2295 bgp_timer_set(peer
);
2297 ret
= FSM_PEER_STOPPED
;
2304 int bgp_gr_lookup_n_update_all_peer(struct bgp
*bgp
,
2305 enum global_mode global_new_state
,
2306 enum global_mode global_old_state
)
2308 struct peer
*peer
= {0};
2309 struct listnode
*node
= {0};
2310 struct listnode
*nnode
= {0};
2311 enum peer_mode peer_old_state
= PEER_INVALID
;
2313 for (ALL_LIST_ELEMENTS(bgp
->peer
, node
, nnode
, peer
)) {
2315 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2316 zlog_debug("%s [BGP_GR] Peer: (%s) :", __func__
,
2319 peer_old_state
= bgp_peer_gr_mode_get(peer
);
2321 if (peer_old_state
== PEER_GLOBAL_INHERIT
) {
2324 *Reset only these peers and send a
2325 *new open message with the change capabilities.
2326 *Considering the mode to be "global_new_state" and
2327 *do all operation accordingly
2330 switch (global_new_state
) {
2332 BGP_PEER_GR_HELPER_ENABLE(peer
);
2335 BGP_PEER_GR_ENABLE(peer
);
2337 case GLOBAL_DISABLE
:
2338 BGP_PEER_GR_DISABLE(peer
);
2340 case GLOBAL_INVALID
:
2341 zlog_debug("%s [BGP_GR] GLOBAL_INVALID",
2343 return BGP_ERR_GR_OPERATION_FAILED
;
2348 bgp
->global_gr_present_state
= global_new_state
;
2350 return BGP_GR_SUCCESS
;
2353 int bgp_gr_update_all(struct bgp
*bgp
, int global_gr_cmd
)
2355 enum global_mode global_new_state
= GLOBAL_INVALID
;
2356 enum global_mode global_old_state
= GLOBAL_INVALID
;
2358 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2359 zlog_debug("%s [BGP_GR]START: global_gr_cmd :%s:", __func__
,
2360 print_global_gr_cmd(global_gr_cmd
));
2362 global_old_state
= bgp_global_gr_mode_get(bgp
);
2364 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2365 zlog_debug("[BGP_GR] global_old_gr_state :%s:",
2366 print_global_gr_mode(global_old_state
));
2368 if (global_old_state
!= GLOBAL_INVALID
) {
2370 bgp
->GLOBAL_GR_FSM
[global_old_state
][global_gr_cmd
];
2372 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2373 zlog_debug("[BGP_GR] global_new_gr_state :%s:",
2374 print_global_gr_mode(global_new_state
));
2376 zlog_err("%s [BGP_GR] global_old_state == GLOBAL_INVALID",
2378 return BGP_ERR_GR_OPERATION_FAILED
;
2381 if (global_new_state
== GLOBAL_INVALID
) {
2382 zlog_err("%s [BGP_GR] global_new_state == GLOBAL_INVALID",
2384 return BGP_ERR_GR_INVALID_CMD
;
2386 if (global_new_state
== global_old_state
) {
2388 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2390 "%s [BGP_GR] global_new_state == global_old_state :%s",
2392 print_global_gr_mode(global_new_state
));
2393 return BGP_GR_NO_OPERATION
;
2396 return bgp_gr_lookup_n_update_all_peer(bgp
, global_new_state
,
2400 const char *print_peer_gr_mode(enum peer_mode pr_mode
)
2402 const char *peer_gr_mode
= NULL
;
2406 peer_gr_mode
= "PEER_HELPER";
2409 peer_gr_mode
= "PEER_GR";
2412 peer_gr_mode
= "PEER_DISABLE";
2415 peer_gr_mode
= "PEER_INVALID";
2417 case PEER_GLOBAL_INHERIT
:
2418 peer_gr_mode
= "PEER_GLOBAL_INHERIT";
2422 return peer_gr_mode
;
2425 const char *print_peer_gr_cmd(enum peer_gr_command pr_gr_cmd
)
2427 const char *peer_gr_cmd
= NULL
;
2429 switch (pr_gr_cmd
) {
2431 peer_gr_cmd
= "PEER_GR_CMD";
2433 case NO_PEER_GR_CMD
:
2434 peer_gr_cmd
= "NO_PEER_GR_CMD";
2436 case PEER_DISABLE_CMD
:
2437 peer_gr_cmd
= "PEER_GR_CMD";
2439 case NO_PEER_DISABLE_CMD
:
2440 peer_gr_cmd
= "NO_PEER_GR_CMD";
2442 case PEER_HELPER_CMD
:
2443 peer_gr_cmd
= "PEER_HELPER_CMD";
2445 case NO_PEER_HELPER_CMD
:
2446 peer_gr_cmd
= "NO_PEER_HELPER_CMD";
2453 const char *print_global_gr_mode(enum global_mode gl_mode
)
2455 const char *global_gr_mode
= NULL
;
2459 global_gr_mode
= "GLOBAL_HELPER";
2462 global_gr_mode
= "GLOBAL_GR";
2464 case GLOBAL_DISABLE
:
2465 global_gr_mode
= "GLOBAL_DISABLE";
2467 case GLOBAL_INVALID
:
2468 global_gr_mode
= "GLOBAL_INVALID";
2472 return global_gr_mode
;
2475 const char *print_global_gr_cmd(enum global_gr_command gl_gr_cmd
)
2477 const char *global_gr_cmd
= NULL
;
2479 switch (gl_gr_cmd
) {
2481 global_gr_cmd
= "GLOBAL_GR_CMD";
2483 case NO_GLOBAL_GR_CMD
:
2484 global_gr_cmd
= "NO_GLOBAL_GR_CMD";
2486 case GLOBAL_DISABLE_CMD
:
2487 global_gr_cmd
= "GLOBAL_DISABLE_CMD";
2489 case NO_GLOBAL_DISABLE_CMD
:
2490 global_gr_cmd
= "NO_GLOBAL_DISABLE_CMD";
2494 return global_gr_cmd
;
2497 enum global_mode
bgp_global_gr_mode_get(struct bgp
*bgp
)
2499 return bgp
->global_gr_present_state
;
2502 enum peer_mode
bgp_peer_gr_mode_get(struct peer
*peer
)
2504 return peer
->peer_gr_present_state
;
2507 int bgp_neighbor_graceful_restart(struct peer
*peer
, int peer_gr_cmd
)
2509 enum peer_mode peer_new_state
= PEER_INVALID
;
2510 enum peer_mode peer_old_state
= PEER_INVALID
;
2511 struct bgp_peer_gr peer_state
;
2512 int result
= BGP_GR_FAILURE
;
2515 * fetch peer_old_state from peer structure also
2516 * fetch global_old_state from bgp structure,
2517 * peer had a back pointer to bgpo struct ;
2520 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2521 zlog_debug("%s [BGP_GR] START:Peer: (%s) : peer_gr_cmd :%s:",
2522 __func__
, peer
->host
,
2523 print_peer_gr_cmd(peer_gr_cmd
));
2525 peer_old_state
= bgp_peer_gr_mode_get(peer
);
2527 if (peer_old_state
== PEER_INVALID
) {
2528 zlog_debug("[BGP_GR] peer_old_state == Invalid state !");
2529 return BGP_ERR_GR_OPERATION_FAILED
;
2532 peer_state
= peer
->PEER_GR_FSM
[peer_old_state
][peer_gr_cmd
];
2533 peer_new_state
= peer_state
.next_state
;
2535 if (peer_new_state
== PEER_INVALID
) {
2537 "[BGP_GR] Invalid bgp graceful restart command used !");
2538 return BGP_ERR_GR_INVALID_CMD
;
2541 if (peer_new_state
!= peer_old_state
) {
2542 result
= peer_state
.action_fun(peer
, peer_old_state
,
2545 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2547 "[BGP_GR] peer_old_state == peer_new_state !");
2548 return BGP_GR_NO_OPERATION
;
2551 if (result
== BGP_GR_SUCCESS
) {
2553 /* Update the mode i.e peer_new_state into the peer structure */
2554 peer
->peer_gr_present_state
= peer_new_state
;
2555 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2557 "[BGP_GR] Successfully change the state of the peer to : %s : !",
2558 print_peer_gr_mode(peer_new_state
));
2560 return BGP_GR_SUCCESS
;
2566 unsigned int bgp_peer_gr_action(struct peer
*peer
, int old_peer_state
,
2569 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2571 "%s [BGP_GR] Move peer from old_peer_state :%s: to new_peer_state :%s: !!!!",
2572 __func__
, print_peer_gr_mode(old_peer_state
),
2573 print_peer_gr_mode(new_peer_state
));
2575 int bgp_gr_global_mode
= GLOBAL_INVALID
;
2576 unsigned int ret
= BGP_GR_FAILURE
;
2578 if (old_peer_state
== new_peer_state
) {
2579 /* Nothing to do over here as the present and old state is the
2581 return BGP_GR_NO_OPERATION
;
2583 if ((old_peer_state
== PEER_INVALID
)
2584 || (new_peer_state
== PEER_INVALID
)) {
2585 /* something bad happend , print error message */
2586 return BGP_ERR_GR_INVALID_CMD
;
2589 bgp_gr_global_mode
= bgp_global_gr_mode_get(peer
->bgp
);
2591 if ((old_peer_state
== PEER_GLOBAL_INHERIT
)
2592 && (new_peer_state
!= PEER_GLOBAL_INHERIT
)) {
2594 /* fetch the Mode running in the Global state machine
2595 *from the bgp structure into a variable called
2599 /* Here we are checking if the
2600 *1. peer_new_state == global_mode == helper_mode
2601 *2. peer_new_state == global_mode == GR_mode
2602 *3. peer_new_state == global_mode == disabled_mode
2605 BGP_PEER_GR_GLOBAL_INHERIT_UNSET(peer
);
2607 if (new_peer_state
== bgp_gr_global_mode
) {
2608 /*This is incremental updates i.e no tear down
2609 *of the existing session
2610 *as the peer is already working in the same mode.
2612 ret
= BGP_GR_SUCCESS
;
2614 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2616 "[BGP_GR] Peer state changed from :%s ",
2617 print_peer_gr_mode(old_peer_state
));
2619 bgp_peer_move_to_gr_mode(peer
, new_peer_state
);
2621 ret
= BGP_GR_SUCCESS
;
2624 /* In the case below peer is going into Global inherit mode i.e.
2625 * the peer would work as the mode configured at the global level
2627 else if ((new_peer_state
== PEER_GLOBAL_INHERIT
)
2628 && (old_peer_state
!= PEER_GLOBAL_INHERIT
)) {
2629 /* Here in this case it would be destructive
2630 * in all the cases except one case when,
2631 * Global GR is configured Disabled
2632 * and present_peer_state is not disable
2635 BGP_PEER_GR_GLOBAL_INHERIT_SET(peer
);
2637 if (old_peer_state
== bgp_gr_global_mode
) {
2639 /* This is incremental updates
2640 *i.e no tear down of the existing session
2641 *as the peer is already working in the same mode.
2643 ret
= BGP_GR_SUCCESS
;
2645 /* Destructive always */
2646 /* Tear down the old session
2647 * and send the new capability
2648 * as per the bgp_gr_global_mode
2651 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2653 "[BGP_GR] Peer state changed from :%s",
2654 print_peer_gr_mode(old_peer_state
));
2656 bgp_peer_move_to_gr_mode(peer
, bgp_gr_global_mode
);
2658 ret
= BGP_GR_SUCCESS
;
2662 *This else case, it include all the cases except -->
2663 *(new_peer_state != Peer_Global) &&
2664 *( old_peer_state != Peer_Global )
2666 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2667 zlog_debug("[BGP_GR] Peer state changed from :%s",
2668 print_peer_gr_mode(old_peer_state
));
2670 bgp_peer_move_to_gr_mode(peer
, new_peer_state
);
2672 ret
= BGP_GR_SUCCESS
;
2678 inline void bgp_peer_move_to_gr_mode(struct peer
*peer
, int new_state
)
2681 int bgp_global_gr_mode
= bgp_global_gr_mode_get(peer
->bgp
);
2683 switch (new_state
) {
2685 BGP_PEER_GR_HELPER_ENABLE(peer
);
2688 BGP_PEER_GR_ENABLE(peer
);
2691 BGP_PEER_GR_DISABLE(peer
);
2693 case PEER_GLOBAL_INHERIT
:
2694 BGP_PEER_GR_GLOBAL_INHERIT_SET(peer
);
2696 if (bgp_global_gr_mode
== GLOBAL_HELPER
) {
2697 BGP_PEER_GR_HELPER_ENABLE(peer
);
2698 } else if (bgp_global_gr_mode
== GLOBAL_GR
) {
2699 BGP_PEER_GR_ENABLE(peer
);
2700 } else if (bgp_global_gr_mode
== GLOBAL_DISABLE
) {
2701 BGP_PEER_GR_DISABLE(peer
);
2704 "[BGP_GR] Default switch inherit mode ::: SOMETHING IS WRONG !!!");
2709 "[BGP_GR] Default switch mode ::: SOMETHING IS WRONG !!!");
2712 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2713 zlog_debug("[BGP_GR] Peer state changed --to--> : %d : !",
2717 void bgp_peer_gr_flags_update(struct peer
*peer
)
2719 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2720 zlog_debug("%s [BGP_GR] called !", __func__
);
2721 if (CHECK_FLAG(peer
->peer_gr_new_status_flag
,
2722 PEER_GRACEFUL_RESTART_NEW_STATE_HELPER
))
2723 SET_FLAG(peer
->flags
, PEER_FLAG_GRACEFUL_RESTART_HELPER
);
2725 UNSET_FLAG(peer
->flags
, PEER_FLAG_GRACEFUL_RESTART_HELPER
);
2726 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2728 "[BGP_GR] Peer %s Flag PEER_FLAG_GRACEFUL_RESTART_HELPER : %s : !",
2730 (CHECK_FLAG(peer
->flags
,
2731 PEER_FLAG_GRACEFUL_RESTART_HELPER
)
2734 if (CHECK_FLAG(peer
->peer_gr_new_status_flag
,
2735 PEER_GRACEFUL_RESTART_NEW_STATE_RESTART
))
2736 SET_FLAG(peer
->flags
, PEER_FLAG_GRACEFUL_RESTART
);
2738 UNSET_FLAG(peer
->flags
, PEER_FLAG_GRACEFUL_RESTART
);
2739 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2741 "[BGP_GR] Peer %s Flag PEER_FLAG_GRACEFUL_RESTART : %s : !",
2743 (CHECK_FLAG(peer
->flags
, PEER_FLAG_GRACEFUL_RESTART
)
2746 if (CHECK_FLAG(peer
->peer_gr_new_status_flag
,
2747 PEER_GRACEFUL_RESTART_NEW_STATE_INHERIT
))
2748 SET_FLAG(peer
->flags
,
2749 PEER_FLAG_GRACEFUL_RESTART_GLOBAL_INHERIT
);
2751 UNSET_FLAG(peer
->flags
,
2752 PEER_FLAG_GRACEFUL_RESTART_GLOBAL_INHERIT
);
2753 if (BGP_DEBUG(graceful_restart
, GRACEFUL_RESTART
))
2755 "[BGP_GR] Peer %s Flag PEER_FLAG_GRACEFUL_RESTART_GLOBAL_INHERIT : %s : !",
2757 (CHECK_FLAG(peer
->flags
,
2758 PEER_FLAG_GRACEFUL_RESTART_GLOBAL_INHERIT
)
2762 if (!CHECK_FLAG(peer
->flags
, PEER_FLAG_GRACEFUL_RESTART
)
2763 && !CHECK_FLAG(peer
->flags
, PEER_FLAG_GRACEFUL_RESTART_HELPER
)) {
2764 zlog_debug("[BGP_GR] Peer %s UNSET PEER_STATUS_NSF_MODE!",
2767 UNSET_FLAG(peer
->sflags
, PEER_STATUS_NSF_MODE
);
2769 if (CHECK_FLAG(peer
->sflags
, PEER_STATUS_NSF_WAIT
)) {
2771 peer_nsf_stop(peer
);
2773 "[BGP_GR] Peer %s UNSET PEER_STATUS_NSF_WAIT!",