]>
Commit | Line | Data |
---|---|---|
718e3744 | 1 | /* BGP network related fucntions |
896014f4 DL |
2 | * Copyright (C) 1999 Kunihiro Ishiguro |
3 | * | |
4 | * This file is part of GNU Zebra. | |
5 | * | |
6 | * GNU Zebra is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License as published by the | |
8 | * Free Software Foundation; either version 2, or (at your option) any | |
9 | * later version. | |
10 | * | |
11 | * GNU Zebra is distributed in the hope that it will be useful, but | |
12 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | * General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License along | |
17 | * with this program; see the file COPYING; if not, write to the Free Software | |
18 | * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | |
19 | */ | |
718e3744 | 20 | |
21 | #include <zebra.h> | |
22 | ||
23 | #include "thread.h" | |
24 | #include "sockunion.h" | |
0df7c91f | 25 | #include "sockopt.h" |
718e3744 | 26 | #include "memory.h" |
27 | #include "log.h" | |
28 | #include "if.h" | |
29 | #include "prefix.h" | |
30 | #include "command.h" | |
edd7c245 | 31 | #include "privs.h" |
0df7c91f | 32 | #include "linklist.h" |
fdbc8e77 | 33 | #include "network.h" |
3f9c7369 | 34 | #include "queue.h" |
66e373ae | 35 | #include "hash.h" |
039f3a34 | 36 | #include "filter.h" |
61cf4b37 | 37 | #include "ns.h" |
174482ef | 38 | #include "lib_errors.h" |
17cdd31e | 39 | #include "nexthop.h" |
718e3744 | 40 | |
41 | #include "bgpd/bgpd.h" | |
1ff9a340 | 42 | #include "bgpd/bgp_open.h" |
718e3744 | 43 | #include "bgpd/bgp_fsm.h" |
44 | #include "bgpd/bgp_attr.h" | |
45 | #include "bgpd/bgp_debug.h" | |
14454c9f | 46 | #include "bgpd/bgp_errors.h" |
718e3744 | 47 | #include "bgpd/bgp_network.h" |
17cdd31e | 48 | #include "bgpd/bgp_zebra.h" |
edd7c245 | 49 | |
50 | extern struct zebra_privs_t bgpd_privs; | |
51 | ||
97896a91 | 52 | static char *bgp_get_bound_name(struct peer *peer); |
f2345335 | 53 | |
d023aec4 | 54 | /* BGP listening socket. */ |
d62a17ae | 55 | struct bgp_listener { |
56 | int fd; | |
57 | union sockunion su; | |
58 | struct thread *thread; | |
61cf4b37 | 59 | struct bgp *bgp; |
2a0e69ae | 60 | char *name; |
d023aec4 | 61 | }; |
6b0655a2 | 62 | |
2a0e69ae DS |
63 | void bgp_dump_listener_info(struct vty *vty) |
64 | { | |
65 | struct listnode *node; | |
66 | struct bgp_listener *listener; | |
67 | ||
68 | vty_out(vty, "Name fd Address\n"); | |
69 | vty_out(vty, "---------------------------\n"); | |
70 | for (ALL_LIST_ELEMENTS_RO(bm->listen_sockets, node, listener)) { | |
71 | char buf[SU_ADDRSTRLEN]; | |
72 | ||
73 | vty_out(vty, "%-16s %d %s\n", | |
74 | listener->name ? listener->name : VRF_DEFAULT_NAME, | |
75 | listener->fd, | |
76 | sockunion2str(&listener->su, buf, sizeof(buf))); | |
77 | } | |
78 | } | |
79 | ||
0df7c91f PJ |
80 | /* |
81 | * Set MD5 key for the socket, for the given IPv4 peer address. | |
82 | * If the password is NULL or zero-length, the option will be disabled. | |
83 | */ | |
d62a17ae | 84 | static int bgp_md5_set_socket(int socket, union sockunion *su, |
9e7d9a61 | 85 | uint16_t prefixlen, const char *password) |
0df7c91f | 86 | { |
d62a17ae | 87 | int ret = -1; |
88 | int en = ENOSYS; | |
ea8b7c71 | 89 | #if HAVE_DECL_TCP_MD5SIG |
d62a17ae | 90 | union sockunion su2; |
ea8b7c71 | 91 | #endif /* HAVE_TCP_MD5SIG */ |
d62a17ae | 92 | |
93 | assert(socket >= 0); | |
94 | ||
95 | #if HAVE_DECL_TCP_MD5SIG | |
96 | /* Ensure there is no extraneous port information. */ | |
97 | memcpy(&su2, su, sizeof(union sockunion)); | |
98 | if (su2.sa.sa_family == AF_INET) | |
99 | su2.sin.sin_port = 0; | |
100 | else | |
101 | su2.sin6.sin6_port = 0; | |
9e7d9a61 QY |
102 | |
103 | /* For addresses, use the non-extended signature functionality */ | |
936fbaef | 104 | if ((su2.sa.sa_family == AF_INET && prefixlen == IPV4_MAX_BITLEN) |
f4d81e55 | 105 | || (su2.sa.sa_family == AF_INET6 && prefixlen == IPV6_MAX_BITLEN)) |
9e7d9a61 QY |
106 | ret = sockopt_tcp_signature(socket, &su2, password); |
107 | else | |
108 | ret = sockopt_tcp_signature_ext(socket, &su2, prefixlen, | |
109 | password); | |
d62a17ae | 110 | en = errno; |
0df7c91f | 111 | #endif /* HAVE_TCP_MD5SIG */ |
0df7c91f | 112 | |
9e7d9a61 QY |
113 | if (ret < 0) { |
114 | char sabuf[SU_ADDRSTRLEN]; | |
115 | sockunion2str(su, sabuf, sizeof(sabuf)); | |
116 | ||
117 | switch (ret) { | |
118 | case -2: | |
119 | flog_warn( | |
120 | EC_BGP_NO_TCP_MD5, | |
121 | "Unable to set TCP MD5 option on socket for peer %s (sock=%d): This platform does not support MD5 auth for prefixes", | |
122 | sabuf, socket); | |
123 | break; | |
124 | default: | |
125 | flog_warn( | |
126 | EC_BGP_NO_TCP_MD5, | |
127 | "Unable to set TCP MD5 option on socket for peer %s (sock=%d): %s", | |
128 | sabuf, socket, safe_strerror(en)); | |
129 | } | |
130 | } | |
d62a17ae | 131 | |
132 | return ret; | |
0df7c91f PJ |
133 | } |
134 | ||
135 | /* Helper for bgp_connect */ | |
d62a17ae | 136 | static int bgp_md5_set_connect(int socket, union sockunion *su, |
9e7d9a61 | 137 | uint16_t prefixlen, const char *password) |
0df7c91f | 138 | { |
d62a17ae | 139 | int ret = -1; |
140 | ||
141 | #if HAVE_DECL_TCP_MD5SIG | |
0cf6db21 | 142 | frr_with_privs(&bgpd_privs) { |
9e7d9a61 | 143 | ret = bgp_md5_set_socket(socket, su, prefixlen, password); |
d62a17ae | 144 | } |
0df7c91f | 145 | #endif /* HAVE_TCP_MD5SIG */ |
d62a17ae | 146 | |
147 | return ret; | |
0df7c91f PJ |
148 | } |
149 | ||
d62a17ae | 150 | static int bgp_md5_set_password(struct peer *peer, const char *password) |
0df7c91f | 151 | { |
d62a17ae | 152 | struct listnode *node; |
153 | int ret = 0; | |
154 | struct bgp_listener *listener; | |
155 | ||
9e7d9a61 QY |
156 | /* |
157 | * Set or unset the password on the listen socket(s). Outbound | |
6bb30c2c | 158 | * connections are taken care of in bgp_connect() below. |
d62a17ae | 159 | */ |
0cf6db21 | 160 | frr_with_privs(&bgpd_privs) { |
6bb30c2c | 161 | for (ALL_LIST_ELEMENTS_RO(bm->listen_sockets, node, listener)) |
a4faae3a PR |
162 | if (listener->su.sa.sa_family == |
163 | peer->su.sa.sa_family) { | |
9e7d9a61 QY |
164 | uint16_t prefixlen = |
165 | peer->su.sa.sa_family == AF_INET | |
936fbaef | 166 | ? IPV4_MAX_BITLEN |
f4d81e55 | 167 | : IPV6_MAX_BITLEN; |
a4faae3a PR |
168 | |
169 | /* | |
170 | * if we have stored a BGP vrf instance in the | |
171 | * listener it must match the bgp instance in | |
172 | * the peer otherwise the peer bgp instance | |
173 | * must be the default vrf or a view instance | |
174 | */ | |
175 | if (!listener->bgp) { | |
2c1eba8e | 176 | if (peer->bgp->vrf_id != VRF_DEFAULT) |
a4faae3a PR |
177 | continue; |
178 | } else if (listener->bgp != peer->bgp) | |
179 | continue; | |
9e7d9a61 | 180 | |
6bb30c2c | 181 | ret = bgp_md5_set_socket(listener->fd, |
9e7d9a61 QY |
182 | &peer->su, prefixlen, |
183 | password); | |
184 | break; | |
185 | } | |
186 | } | |
187 | return ret; | |
188 | } | |
189 | ||
a4faae3a | 190 | int bgp_md5_set_prefix(struct bgp *bgp, struct prefix *p, const char *password) |
9e7d9a61 QY |
191 | { |
192 | int ret = 0; | |
193 | union sockunion su; | |
194 | struct listnode *node; | |
195 | struct bgp_listener *listener; | |
196 | ||
197 | /* Set or unset the password on the listen socket(s). */ | |
0cf6db21 | 198 | frr_with_privs(&bgpd_privs) { |
9e7d9a61 | 199 | for (ALL_LIST_ELEMENTS_RO(bm->listen_sockets, node, listener)) |
a4faae3a PR |
200 | if (listener->su.sa.sa_family == p->family |
201 | && ((bgp->vrf_id == VRF_DEFAULT) | |
202 | || (listener->bgp == bgp))) { | |
9e7d9a61 QY |
203 | prefix2sockunion(p, &su); |
204 | ret = bgp_md5_set_socket(listener->fd, &su, | |
205 | p->prefixlen, | |
206 | password); | |
6bb30c2c DL |
207 | break; |
208 | } | |
209 | } | |
9e7d9a61 | 210 | |
d62a17ae | 211 | return ret; |
0df7c91f | 212 | } |
3374bef0 | 213 | |
a4faae3a | 214 | int bgp_md5_unset_prefix(struct bgp *bgp, struct prefix *p) |
9e7d9a61 | 215 | { |
a4faae3a | 216 | return bgp_md5_set_prefix(bgp, p, NULL); |
9e7d9a61 QY |
217 | } |
218 | ||
d62a17ae | 219 | int bgp_md5_set(struct peer *peer) |
89ca90fa | 220 | { |
d62a17ae | 221 | /* Set the password from listen socket. */ |
222 | return bgp_md5_set_password(peer, peer->password); | |
89ca90fa | 223 | } |
224 | ||
d62a17ae | 225 | int bgp_md5_unset(struct peer *peer) |
89ca90fa | 226 | { |
d62a17ae | 227 | /* Unset the password from listen socket. */ |
228 | return bgp_md5_set_password(peer, NULL); | |
89ca90fa | 229 | } |
230 | ||
d62a17ae | 231 | int bgp_set_socket_ttl(struct peer *peer, int bgp_sock) |
ef0b0c3e | 232 | { |
d62a17ae | 233 | int ret = 0; |
234 | ||
235 | /* In case of peer is EBGP, we should set TTL for this connection. */ | |
bf0d28dc | 236 | if (!peer->gtsm_hops && (peer_sort_lookup(peer) == BGP_PEER_EBGP)) { |
d62a17ae | 237 | ret = sockopt_ttl(peer->su.sa.sa_family, bgp_sock, peer->ttl); |
238 | if (ret) { | |
af4c2728 | 239 | flog_err( |
450971aa | 240 | EC_LIB_SOCKET, |
c0d72166 DS |
241 | "%s: Can't set TxTTL on peer (rtrid %pI4) socket, err = %d", |
242 | __func__, &peer->remote_id, errno); | |
d62a17ae | 243 | return ret; |
244 | } | |
245 | } else if (peer->gtsm_hops) { | |
246 | /* On Linux, setting minttl without setting ttl seems to mess | |
247 | with the | |
248 | outgoing ttl. Therefore setting both. | |
249 | */ | |
250 | ret = sockopt_ttl(peer->su.sa.sa_family, bgp_sock, MAXTTL); | |
251 | if (ret) { | |
af4c2728 | 252 | flog_err( |
450971aa | 253 | EC_LIB_SOCKET, |
c0d72166 DS |
254 | "%s: Can't set TxTTL on peer (rtrid %pI4) socket, err = %d", |
255 | __func__, &peer->remote_id, errno); | |
d62a17ae | 256 | return ret; |
257 | } | |
258 | ret = sockopt_minttl(peer->su.sa.sa_family, bgp_sock, | |
259 | MAXTTL + 1 - peer->gtsm_hops); | |
260 | if (ret) { | |
af4c2728 | 261 | flog_err( |
450971aa | 262 | EC_LIB_SOCKET, |
c0d72166 DS |
263 | "%s: Can't set MinTTL on peer (rtrid %pI4) socket, err = %d", |
264 | __func__, &peer->remote_id, errno); | |
d62a17ae | 265 | return ret; |
266 | } | |
267 | } | |
268 | ||
269 | return ret; | |
ef0b0c3e DL |
270 | } |
271 | ||
8dee0396 | 272 | /* |
273 | * Obtain the BGP instance that the incoming connection should be processed | |
274 | * against. This is important because more than one VRF could be using the | |
275 | * same IP address space. The instance is got by obtaining the device to | |
276 | * which the incoming connection is bound to. This could either be a VRF | |
277 | * or it could be an interface, which in turn determines the VRF. | |
278 | */ | |
d62a17ae | 279 | static int bgp_get_instance_for_inc_conn(int sock, struct bgp **bgp_inst) |
8dee0396 | 280 | { |
120c6587 | 281 | #ifndef SO_BINDTODEVICE |
d62a17ae | 282 | /* only Linux has SO_BINDTODEVICE, but we're in Linux-specific code here |
283 | * anyway since the assumption is that the interface name returned by | |
284 | * getsockopt() is useful in identifying the VRF, particularly with | |
285 | * Linux's | |
286 | * VRF l3master device. The whole mechanism is specific to Linux, so... | |
287 | * when other platforms add VRF support, this will need handling here as | |
288 | * well. (or, some restructuring) */ | |
289 | *bgp_inst = bgp_get_default(); | |
290 | return !*bgp_inst; | |
120c6587 DL |
291 | |
292 | #else | |
d62a17ae | 293 | char name[VRF_NAMSIZ + 1]; |
294 | socklen_t name_len = VRF_NAMSIZ; | |
295 | struct bgp *bgp; | |
296 | int rc; | |
297 | struct listnode *node, *nnode; | |
298 | ||
299 | *bgp_inst = NULL; | |
300 | name[0] = '\0'; | |
301 | rc = getsockopt(sock, SOL_SOCKET, SO_BINDTODEVICE, name, &name_len); | |
302 | if (rc != 0) { | |
303 | #if defined(HAVE_CUMULUS) | |
1c50c1c0 QY |
304 | flog_err(EC_LIB_SOCKET, |
305 | "[Error] BGP SO_BINDTODEVICE get failed (%s), sock %d", | |
306 | safe_strerror(errno), sock); | |
d62a17ae | 307 | return -1; |
b5826a12 | 308 | #endif |
d62a17ae | 309 | } |
310 | ||
311 | if (!strlen(name)) { | |
312 | *bgp_inst = bgp_get_default(); | |
313 | return 0; /* default instance. */ | |
314 | } | |
315 | ||
316 | /* First try match to instance; if that fails, check for interfaces. */ | |
317 | bgp = bgp_lookup_by_name(name); | |
318 | if (bgp) { | |
319 | if (!bgp->vrf_id) // unexpected | |
320 | return -1; | |
321 | *bgp_inst = bgp; | |
322 | return 0; | |
323 | } | |
324 | ||
325 | /* TODO - This will be optimized once interfaces move into the NS */ | |
326 | for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) { | |
327 | struct interface *ifp; | |
328 | ||
329 | if (bgp->inst_type == BGP_INSTANCE_TYPE_VIEW) | |
330 | continue; | |
331 | ||
a36898e7 | 332 | ifp = if_lookup_by_name(name, bgp->vrf_id); |
d62a17ae | 333 | if (ifp) { |
334 | *bgp_inst = bgp; | |
335 | return 0; | |
336 | } | |
337 | } | |
338 | ||
339 | /* We didn't match to either an instance or an interface. */ | |
340 | return -1; | |
120c6587 | 341 | #endif |
8dee0396 | 342 | } |
343 | ||
c2d020ad DS |
344 | static void bgp_socket_set_buffer_size(const int fd) |
345 | { | |
346 | if (getsockopt_so_sendbuf(fd) < (int)bm->socket_buffer) | |
347 | setsockopt_so_sendbuf(fd, bm->socket_buffer); | |
348 | if (getsockopt_so_recvbuf(fd) < (int)bm->socket_buffer) | |
349 | setsockopt_so_recvbuf(fd, bm->socket_buffer); | |
350 | } | |
351 | ||
718e3744 | 352 | /* Accept bgp connection. */ |
d62a17ae | 353 | static int bgp_accept(struct thread *thread) |
718e3744 | 354 | { |
d62a17ae | 355 | int bgp_sock; |
356 | int accept_sock; | |
357 | union sockunion su; | |
358 | struct bgp_listener *listener = THREAD_ARG(thread); | |
359 | struct peer *peer; | |
360 | struct peer *peer1; | |
361 | char buf[SU_ADDRSTRLEN]; | |
362 | struct bgp *bgp = NULL; | |
363 | ||
364 | sockunion_init(&su); | |
365 | ||
2f9db11b QY |
366 | bgp = bgp_lookup_by_name(listener->name); |
367 | ||
d62a17ae | 368 | /* Register accept thread. */ |
369 | accept_sock = THREAD_FD(thread); | |
370 | if (accept_sock < 0) { | |
2f9db11b QY |
371 | flog_err_sys(EC_LIB_SOCKET, |
372 | "[Error] BGP accept socket fd is negative: %d", | |
09c866e3 | 373 | accept_sock); |
d62a17ae | 374 | return -1; |
375 | } | |
61cf4b37 | 376 | |
d62a17ae | 377 | thread_add_read(bm->master, bgp_accept, listener, accept_sock, |
378 | &listener->thread); | |
379 | ||
380 | /* Accept client connection. */ | |
381 | bgp_sock = sockunion_accept(accept_sock, &su); | |
2f9db11b | 382 | int save_errno = errno; |
d62a17ae | 383 | if (bgp_sock < 0) { |
2f9db11b QY |
384 | if (save_errno == EINVAL) { |
385 | struct vrf *vrf = | |
386 | bgp ? vrf_lookup_by_id(bgp->vrf_id) : NULL; | |
387 | ||
388 | /* | |
389 | * It appears that sometimes, when VRFs are deleted on | |
390 | * the system, it takes a little while for us to get | |
391 | * notified about that. In the meantime we endlessly | |
392 | * loop on accept(), because the socket, having been | |
393 | * bound to a now-deleted VRF device, is in some weird | |
394 | * state which causes accept() to fail. | |
395 | * | |
396 | * To avoid this, if we see accept() fail with EINVAL, | |
397 | * we cancel ourselves and trust that when the VRF | |
398 | * deletion notification comes in the event handler for | |
399 | * that will take care of cleaning us up. | |
400 | */ | |
401 | flog_err_sys( | |
402 | EC_LIB_SOCKET, | |
403 | "[Error] accept() failed with error \"%s\" on BGP listener socket %d for BGP instance in VRF \"%s\"; refreshing socket", | |
404 | safe_strerror(save_errno), accept_sock, | |
405 | VRF_LOGNAME(vrf)); | |
406 | THREAD_OFF(listener->thread); | |
407 | } else { | |
408 | flog_err_sys( | |
409 | EC_LIB_SOCKET, | |
410 | "[Error] BGP socket accept failed (%s); retrying", | |
411 | safe_strerror(save_errno)); | |
412 | } | |
d62a17ae | 413 | return -1; |
414 | } | |
415 | set_nonblocking(bgp_sock); | |
416 | ||
61cf4b37 PG |
417 | /* Obtain BGP instance this connection is meant for. |
418 | * - if it is a VRF netns sock, then BGP is in listener structure | |
419 | * - otherwise, the bgp instance need to be demultiplexed | |
420 | */ | |
421 | if (listener->bgp) | |
422 | bgp = listener->bgp; | |
423 | else if (bgp_get_instance_for_inc_conn(bgp_sock, &bgp)) { | |
d62a17ae | 424 | if (bgp_debug_neighbor_events(NULL)) |
425 | zlog_debug( | |
426 | "[Event] Could not get instance for incoming conn from %s", | |
427 | inet_sutop(&su, buf)); | |
428 | close(bgp_sock); | |
429 | return -1; | |
430 | } | |
431 | ||
c2d020ad | 432 | bgp_socket_set_buffer_size(bgp_sock); |
d62a17ae | 433 | |
434 | /* Check remote IP address */ | |
435 | peer1 = peer_lookup(bgp, &su); | |
436 | ||
437 | if (!peer1) { | |
438 | peer1 = peer_lookup_dynamic_neighbor(bgp, &su); | |
439 | if (peer1) { | |
440 | /* Dynamic neighbor has been created, let it proceed */ | |
441 | peer1->fd = bgp_sock; | |
4ab46701 AR |
442 | |
443 | /* Set the user configured MSS to TCP socket */ | |
444 | if (CHECK_FLAG(peer1->flags, PEER_FLAG_TCP_MSS)) | |
445 | sockopt_tcp_mss_set(bgp_sock, peer1->tcp_mss); | |
446 | ||
d62a17ae | 447 | bgp_fsm_change_status(peer1, Active); |
448 | BGP_TIMER_OFF( | |
449 | peer1->t_start); /* created in peer_create() */ | |
450 | ||
6c537a18 DS |
451 | if (peer_active(peer1)) { |
452 | if (CHECK_FLAG(peer1->flags, | |
453 | PEER_FLAG_TIMER_DELAYOPEN)) | |
454 | BGP_EVENT_ADD( | |
455 | peer1, | |
456 | TCP_connection_open_w_delay); | |
457 | else | |
458 | BGP_EVENT_ADD(peer1, | |
459 | TCP_connection_open); | |
460 | } | |
d62a17ae | 461 | |
462 | return 0; | |
463 | } | |
464 | } | |
465 | ||
466 | if (!peer1) { | |
467 | if (bgp_debug_neighbor_events(NULL)) { | |
468 | zlog_debug( | |
45069622 DS |
469 | "[Event] %s connection rejected(%s:%u:%s) - not configured and not valid for dynamic", |
470 | inet_sutop(&su, buf), bgp->name_pretty, bgp->as, | |
471 | VRF_LOGNAME(vrf_lookup_by_id(bgp->vrf_id))); | |
d62a17ae | 472 | } |
473 | close(bgp_sock); | |
474 | return -1; | |
475 | } | |
476 | ||
cb9196e7 DS |
477 | if (CHECK_FLAG(peer1->flags, PEER_FLAG_SHUTDOWN) |
478 | || CHECK_FLAG(peer1->bgp->flags, BGP_FLAG_SHUTDOWN)) { | |
d62a17ae | 479 | if (bgp_debug_neighbor_events(peer1)) |
480 | zlog_debug( | |
45069622 DS |
481 | "[Event] connection from %s rejected(%s:%u:%s) due to admin shutdown", |
482 | inet_sutop(&su, buf), bgp->name_pretty, bgp->as, | |
483 | VRF_LOGNAME(vrf_lookup_by_id(bgp->vrf_id))); | |
d62a17ae | 484 | close(bgp_sock); |
485 | return -1; | |
486 | } | |
487 | ||
488 | /* | |
489 | * Do not accept incoming connections in Clearing state. This can result | |
490 | * in incorect state transitions - e.g., the connection goes back to | |
491 | * Established and then the Clearing_Completed event is generated. Also, | |
492 | * block incoming connection in Deleted state. | |
493 | */ | |
494 | if (peer1->status == Clearing || peer1->status == Deleted) { | |
495 | if (bgp_debug_neighbor_events(peer1)) | |
496 | zlog_debug( | |
497 | "[Event] Closing incoming conn for %s (%p) state %d", | |
498 | peer1->host, peer1, peer1->status); | |
499 | close(bgp_sock); | |
500 | return -1; | |
501 | } | |
502 | ||
503 | /* Check that at least one AF is activated for the peer. */ | |
504 | if (!peer_active(peer1)) { | |
505 | if (bgp_debug_neighbor_events(peer1)) | |
506 | zlog_debug( | |
507 | "%s - incoming conn rejected - no AF activated for peer", | |
508 | peer1->host); | |
509 | close(bgp_sock); | |
510 | return -1; | |
511 | } | |
512 | ||
d091d9ad DA |
513 | /* Do not try to reconnect if the peer reached maximum |
514 | * prefixes, restart timer is still running or the peer | |
515 | * is shutdown. | |
516 | */ | |
517 | if (BGP_PEER_START_SUPPRESSED(peer1)) { | |
497b686a MS |
518 | if (bgp_debug_neighbor_events(peer1)) |
519 | zlog_debug( | |
3efd0893 | 520 | "[Event] Incoming BGP connection rejected from %s due to maximum-prefix or shutdown", |
497b686a MS |
521 | peer1->host); |
522 | close(bgp_sock); | |
523 | return -1; | |
524 | } | |
525 | ||
d62a17ae | 526 | if (bgp_debug_neighbor_events(peer1)) |
527 | zlog_debug("[Event] BGP connection from host %s fd %d", | |
528 | inet_sutop(&su, buf), bgp_sock); | |
529 | ||
530 | if (peer1->doppelganger) { | |
531 | /* We have an existing connection. Kill the existing one and run | |
532 | with this one. | |
533 | */ | |
534 | if (bgp_debug_neighbor_events(peer1)) | |
535 | zlog_debug( | |
3efd0893 | 536 | "[Event] New active connection from peer %s, Killing previous active connection", |
d62a17ae | 537 | peer1->host); |
538 | peer_delete(peer1->doppelganger); | |
539 | } | |
540 | ||
541 | if (bgp_set_socket_ttl(peer1, bgp_sock) < 0) | |
542 | if (bgp_debug_neighbor_events(peer1)) | |
543 | zlog_debug( | |
544 | "[Event] Unable to set min/max TTL on peer %s, Continuing", | |
545 | peer1->host); | |
546 | ||
547 | peer = peer_create(&su, peer1->conf_if, peer1->bgp, peer1->local_as, | |
e84c59af | 548 | peer1->as, peer1->as_type, NULL); |
d62a17ae | 549 | hash_release(peer->bgp->peerhash, peer); |
550 | hash_get(peer->bgp->peerhash, peer, hash_alloc_intern); | |
551 | ||
552 | peer_xfer_config(peer, peer1); | |
d7b3cda6 | 553 | bgp_peer_gr_flags_update(peer); |
554 | ||
36235319 QY |
555 | BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(peer->bgp, |
556 | peer->bgp->peer); | |
0f0444fb | 557 | |
d7b3cda6 | 558 | if (bgp_peer_gr_mode_get(peer) == PEER_DISABLE) { |
559 | ||
560 | UNSET_FLAG(peer->sflags, PEER_STATUS_NSF_MODE); | |
561 | ||
36235319 | 562 | if (CHECK_FLAG(peer->sflags, PEER_STATUS_NSF_WAIT)) { |
d7b3cda6 | 563 | peer_nsf_stop(peer); |
564 | } | |
565 | } | |
566 | ||
d62a17ae | 567 | UNSET_FLAG(peer->flags, PEER_FLAG_CONFIG_NODE); |
568 | ||
569 | peer->doppelganger = peer1; | |
570 | peer1->doppelganger = peer; | |
571 | peer->fd = bgp_sock; | |
0def0c9f IR |
572 | frr_with_privs(&bgpd_privs) { |
573 | vrf_bind(peer->bgp->vrf_id, bgp_sock, bgp_get_bound_name(peer)); | |
574 | } | |
996319e6 | 575 | bgp_peer_reg_with_nht(peer); |
d62a17ae | 576 | bgp_fsm_change_status(peer, Active); |
577 | BGP_TIMER_OFF(peer->t_start); /* created in peer_create() */ | |
578 | ||
579 | SET_FLAG(peer->sflags, PEER_STATUS_ACCEPT_PEER); | |
d62a17ae | 580 | /* Make dummy peer until read Open packet. */ |
feb17238 | 581 | if (peer_established(peer1) |
36235319 | 582 | && CHECK_FLAG(peer1->sflags, PEER_STATUS_NSF_MODE)) { |
d62a17ae | 583 | /* If we have an existing established connection with graceful |
584 | * restart | |
585 | * capability announced with one or more address families, then | |
586 | * drop | |
587 | * existing established connection and move state to connect. | |
588 | */ | |
589 | peer1->last_reset = PEER_DOWN_NSF_CLOSE_SESSION; | |
d7b3cda6 | 590 | |
36235319 QY |
591 | if (CHECK_FLAG(peer1->flags, PEER_FLAG_GRACEFUL_RESTART) |
592 | || CHECK_FLAG(peer1->flags, | |
593 | PEER_FLAG_GRACEFUL_RESTART_HELPER)) | |
594 | SET_FLAG(peer1->sflags, PEER_STATUS_NSF_WAIT); | |
d7b3cda6 | 595 | |
d62a17ae | 596 | bgp_event_update(peer1, TCP_connection_closed); |
597 | } | |
598 | ||
599 | if (peer_active(peer)) { | |
6c537a18 DS |
600 | if (CHECK_FLAG(peer->flags, PEER_FLAG_TIMER_DELAYOPEN)) |
601 | BGP_EVENT_ADD(peer, TCP_connection_open_w_delay); | |
602 | else | |
603 | BGP_EVENT_ADD(peer, TCP_connection_open); | |
d62a17ae | 604 | } |
605 | ||
606 | return 0; | |
718e3744 | 607 | } |
608 | ||
609 | /* BGP socket bind. */ | |
97896a91 | 610 | static char *bgp_get_bound_name(struct peer *peer) |
718e3744 | 611 | { |
a2bce1c8 DS |
612 | if (!peer) |
613 | return NULL; | |
614 | ||
996c9314 LB |
615 | if ((peer->bgp->vrf_id == VRF_DEFAULT) && !peer->ifname |
616 | && !peer->conf_if) | |
97896a91 PG |
617 | return NULL; |
618 | ||
d62a17ae | 619 | if (peer->su.sa.sa_family != AF_INET |
620 | && peer->su.sa.sa_family != AF_INET6) | |
97896a91 | 621 | return NULL; // unexpected |
d62a17ae | 622 | |
623 | /* For IPv6 peering, interface (unnumbered or link-local with interface) | |
624 | * takes precedence over VRF. For IPv4 peering, explicit interface or | |
625 | * VRF are the situations to bind. | |
626 | */ | |
7224dcd8 IR |
627 | if (peer->su.sa.sa_family == AF_INET6 && peer->conf_if) |
628 | return peer->conf_if; | |
629 | ||
630 | if (peer->ifname) | |
631 | return peer->ifname; | |
632 | ||
633 | if (peer->bgp->inst_type == BGP_INSTANCE_TYPE_VIEW) | |
634 | return NULL; | |
d62a17ae | 635 | |
7224dcd8 | 636 | return peer->bgp->name; |
718e3744 | 637 | } |
638 | ||
d62a17ae | 639 | static int bgp_update_address(struct interface *ifp, const union sockunion *dst, |
640 | union sockunion *addr) | |
718e3744 | 641 | { |
d62a17ae | 642 | struct prefix *p, *sel, d; |
643 | struct connected *connected; | |
644 | struct listnode *node; | |
645 | int common; | |
646 | ||
0154d8ce DS |
647 | if (!sockunion2hostprefix(dst, &d)) |
648 | return 1; | |
649 | ||
d62a17ae | 650 | sel = NULL; |
651 | common = -1; | |
652 | ||
653 | for (ALL_LIST_ELEMENTS_RO(ifp->connected, node, connected)) { | |
654 | p = connected->address; | |
655 | if (p->family != d.family) | |
656 | continue; | |
657 | if (prefix_common_bits(p, &d) > common) { | |
658 | sel = p; | |
659 | common = prefix_common_bits(sel, &d); | |
660 | } | |
661 | } | |
662 | ||
663 | if (!sel) | |
664 | return 1; | |
665 | ||
666 | prefix2sockunion(sel, addr); | |
667 | return 0; | |
718e3744 | 668 | } |
669 | ||
670 | /* Update source selection. */ | |
d62a17ae | 671 | static int bgp_update_source(struct peer *peer) |
718e3744 | 672 | { |
d62a17ae | 673 | struct interface *ifp; |
674 | union sockunion addr; | |
675 | int ret = 0; | |
718e3744 | 676 | |
d62a17ae | 677 | sockunion_init(&addr); |
dd793e4a | 678 | |
d62a17ae | 679 | /* Source is specified with interface name. */ |
680 | if (peer->update_if) { | |
a36898e7 | 681 | ifp = if_lookup_by_name(peer->update_if, peer->bgp->vrf_id); |
d62a17ae | 682 | if (!ifp) |
683 | return -1; | |
718e3744 | 684 | |
d62a17ae | 685 | if (bgp_update_address(ifp, &peer->su, &addr)) |
686 | return -1; | |
718e3744 | 687 | |
d62a17ae | 688 | ret = sockunion_bind(peer->fd, &addr, 0, &addr); |
689 | } | |
718e3744 | 690 | |
d62a17ae | 691 | /* Source is specified with IP address. */ |
692 | if (peer->update_source) | |
693 | ret = sockunion_bind(peer->fd, peer->update_source, 0, | |
694 | peer->update_source); | |
49067496 | 695 | |
d62a17ae | 696 | return ret; |
718e3744 | 697 | } |
698 | ||
699 | /* BGP try to connect to the peer. */ | |
d62a17ae | 700 | int bgp_connect(struct peer *peer) |
718e3744 | 701 | { |
b750b0ba QY |
702 | assert(!CHECK_FLAG(peer->thread_flags, PEER_THREAD_WRITES_ON)); |
703 | assert(!CHECK_FLAG(peer->thread_flags, PEER_THREAD_READS_ON)); | |
d62a17ae | 704 | ifindex_t ifindex = 0; |
705 | ||
706 | if (peer->conf_if && BGP_PEER_SU_UNSPEC(peer)) { | |
888e727c DS |
707 | if (bgp_debug_neighbor_events(peer)) |
708 | zlog_debug("Peer address not learnt: Returning from connect"); | |
d62a17ae | 709 | return 0; |
710 | } | |
0cf6db21 | 711 | frr_with_privs(&bgpd_privs) { |
d62a17ae | 712 | /* Make socket for the peer. */ |
01b9e3fd DL |
713 | peer->fd = vrf_sockunion_socket(&peer->su, peer->bgp->vrf_id, |
714 | bgp_get_bound_name(peer)); | |
715 | } | |
888e727c DS |
716 | if (peer->fd < 0) { |
717 | if (bgp_debug_neighbor_events(peer)) | |
718 | zlog_debug("%s: Failure to create socket for connection to %s, error received: %s(%d)", | |
719 | __func__, peer->host, safe_strerror(errno), | |
720 | errno); | |
d62a17ae | 721 | return -1; |
888e727c | 722 | } |
d62a17ae | 723 | |
724 | set_nonblocking(peer->fd); | |
725 | ||
4ab46701 AR |
726 | /* Set the user configured MSS to TCP socket */ |
727 | if (CHECK_FLAG(peer->flags, PEER_FLAG_TCP_MSS)) | |
728 | sockopt_tcp_mss_set(peer->fd, peer->tcp_mss); | |
729 | ||
c2d020ad | 730 | bgp_socket_set_buffer_size(peer->fd); |
d62a17ae | 731 | |
888e727c DS |
732 | if (bgp_set_socket_ttl(peer, peer->fd) < 0) { |
733 | if (bgp_debug_neighbor_events(peer)) | |
734 | zlog_debug("%s: Failure to set socket ttl for connection to %s, error received: %s(%d)", | |
735 | __func__, peer->host, safe_strerror(errno), | |
736 | errno); | |
d62a17ae | 737 | return -1; |
888e727c | 738 | } |
d62a17ae | 739 | |
740 | sockopt_reuseaddr(peer->fd); | |
741 | sockopt_reuseport(peer->fd); | |
d62a17ae | 742 | |
1423c809 | 743 | #ifdef IPTOS_PREC_INTERNETCONTROL |
0cf6db21 | 744 | frr_with_privs(&bgpd_privs) { |
01b9e3fd | 745 | if (sockunion_family(&peer->su) == AF_INET) |
633fc9b1 DL |
746 | setsockopt_ipv4_tos(peer->fd, |
747 | IPTOS_PREC_INTERNETCONTROL); | |
01b9e3fd | 748 | else if (sockunion_family(&peer->su) == AF_INET6) |
633fc9b1 DL |
749 | setsockopt_ipv6_tclass(peer->fd, |
750 | IPTOS_PREC_INTERNETCONTROL); | |
01b9e3fd | 751 | } |
1423c809 SH |
752 | #endif |
753 | ||
9e7d9a61 QY |
754 | if (peer->password) { |
755 | uint16_t prefixlen = peer->su.sa.sa_family == AF_INET | |
936fbaef | 756 | ? IPV4_MAX_BITLEN |
f4d81e55 | 757 | : IPV6_MAX_BITLEN; |
9e7d9a61 QY |
758 | |
759 | bgp_md5_set_connect(peer->fd, &peer->su, prefixlen, | |
760 | peer->password); | |
761 | } | |
718e3744 | 762 | |
d62a17ae | 763 | /* Update source bind. */ |
764 | if (bgp_update_source(peer) < 0) { | |
765 | return connect_error; | |
766 | } | |
718e3744 | 767 | |
d62a17ae | 768 | if (peer->conf_if || peer->ifname) |
769 | ifindex = ifname2ifindex(peer->conf_if ? peer->conf_if | |
770 | : peer->ifname, | |
771 | peer->bgp->vrf_id); | |
718e3744 | 772 | |
d62a17ae | 773 | if (bgp_debug_neighbor_events(peer)) |
774 | zlog_debug("%s [Event] Connect start to %s fd %d", peer->host, | |
775 | peer->host, peer->fd); | |
718e3744 | 776 | |
d62a17ae | 777 | /* Connect to the remote peer. */ |
778 | return sockunion_connect(peer->fd, &peer->su, htons(peer->port), | |
779 | ifindex); | |
718e3744 | 780 | } |
781 | ||
782 | /* After TCP connection is established. Get local address and port. */ | |
d62a17ae | 783 | int bgp_getsockname(struct peer *peer) |
718e3744 | 784 | { |
d62a17ae | 785 | if (peer->su_local) { |
786 | sockunion_free(peer->su_local); | |
787 | peer->su_local = NULL; | |
788 | } | |
789 | ||
790 | if (peer->su_remote) { | |
791 | sockunion_free(peer->su_remote); | |
792 | peer->su_remote = NULL; | |
793 | } | |
794 | ||
795 | peer->su_local = sockunion_getsockname(peer->fd); | |
796 | if (!peer->su_local) | |
797 | return -1; | |
798 | peer->su_remote = sockunion_getpeername(peer->fd); | |
799 | if (!peer->su_remote) | |
800 | return -1; | |
801 | ||
17cdd31e DS |
802 | if (!bgp_zebra_nexthop_set(peer->su_local, peer->su_remote, |
803 | &peer->nexthop, peer)) { | |
2564f080 | 804 | flog_err(EC_BGP_NH_UPD, |
17cdd31e DS |
805 | "%s: nexthop_set failed, resetting connection - intf %p", |
806 | peer->host, peer->nexthop.ifp); | |
d62a17ae | 807 | return -1; |
d62a17ae | 808 | } |
d62a17ae | 809 | return 0; |
718e3744 | 810 | } |
811 | ||
d023aec4 | 812 | |
61cf4b37 PG |
813 | static int bgp_listener(int sock, struct sockaddr *sa, socklen_t salen, |
814 | struct bgp *bgp) | |
d023aec4 | 815 | { |
d62a17ae | 816 | struct bgp_listener *listener; |
817 | int ret, en; | |
d023aec4 | 818 | |
d62a17ae | 819 | sockopt_reuseaddr(sock); |
820 | sockopt_reuseport(sock); | |
d023aec4 | 821 | |
0cf6db21 | 822 | frr_with_privs(&bgpd_privs) { |
5c88f19d | 823 | |
d023aec4 | 824 | #ifdef IPTOS_PREC_INTERNETCONTROL |
01b9e3fd DL |
825 | if (sa->sa_family == AF_INET) |
826 | setsockopt_ipv4_tos(sock, IPTOS_PREC_INTERNETCONTROL); | |
827 | else if (sa->sa_family == AF_INET6) | |
633fc9b1 DL |
828 | setsockopt_ipv6_tclass(sock, |
829 | IPTOS_PREC_INTERNETCONTROL); | |
d023aec4 SH |
830 | #endif |
831 | ||
01b9e3fd | 832 | sockopt_v6only(sa->sa_family, sock); |
d62a17ae | 833 | |
01b9e3fd DL |
834 | ret = bind(sock, sa, salen); |
835 | en = errno; | |
836 | } | |
d62a17ae | 837 | |
838 | if (ret < 0) { | |
450971aa | 839 | flog_err_sys(EC_LIB_SOCKET, "bind: %s", safe_strerror(en)); |
d62a17ae | 840 | return ret; |
841 | } | |
842 | ||
48522088 | 843 | ret = listen(sock, SOMAXCONN); |
d62a17ae | 844 | if (ret < 0) { |
1c50c1c0 | 845 | flog_err_sys(EC_LIB_SOCKET, "listen: %s", safe_strerror(errno)); |
d62a17ae | 846 | return ret; |
847 | } | |
848 | ||
61cf4b37 | 849 | listener = XCALLOC(MTYPE_BGP_LISTENER, sizeof(*listener)); |
d62a17ae | 850 | listener->fd = sock; |
2a0e69ae | 851 | listener->name = XSTRDUP(MTYPE_BGP_LISTENER, bgp->name); |
61cf4b37 | 852 | |
a4faae3a | 853 | /* this socket is in a vrf record bgp back pointer */ |
2c1eba8e | 854 | if (bgp->vrf_id != VRF_DEFAULT) |
61cf4b37 PG |
855 | listener->bgp = bgp; |
856 | ||
d62a17ae | 857 | memcpy(&listener->su, sa, salen); |
d62a17ae | 858 | thread_add_read(bm->master, bgp_accept, listener, sock, |
859 | &listener->thread); | |
860 | listnode_add(bm->listen_sockets, listener); | |
861 | ||
862 | return 0; | |
d023aec4 SH |
863 | } |
864 | ||
718e3744 | 865 | /* IPv6 supported version of BGP server socket setup. */ |
61cf4b37 | 866 | int bgp_socket(struct bgp *bgp, unsigned short port, const char *address) |
718e3744 | 867 | { |
d62a17ae | 868 | struct addrinfo *ainfo; |
869 | struct addrinfo *ainfo_save; | |
870 | static const struct addrinfo req = { | |
871 | .ai_family = AF_UNSPEC, | |
872 | .ai_flags = AI_PASSIVE, | |
873 | .ai_socktype = SOCK_STREAM, | |
874 | }; | |
875 | int ret, count; | |
876 | char port_str[BUFSIZ]; | |
877 | ||
878 | snprintf(port_str, sizeof(port_str), "%d", port); | |
879 | port_str[sizeof(port_str) - 1] = '\0'; | |
880 | ||
0cf6db21 | 881 | frr_with_privs(&bgpd_privs) { |
01b9e3fd DL |
882 | ret = vrf_getaddrinfo(address, port_str, &req, &ainfo_save, |
883 | bgp->vrf_id); | |
884 | } | |
d62a17ae | 885 | if (ret != 0) { |
450971aa | 886 | flog_err_sys(EC_LIB_SOCKET, "getaddrinfo: %s", |
09c866e3 | 887 | gai_strerror(ret)); |
d62a17ae | 888 | return -1; |
889 | } | |
0b014ea6 | 890 | if (bgp_option_check(BGP_OPT_NO_ZEBRA) && |
449cff3e A |
891 | bgp->vrf_id != VRF_DEFAULT) { |
892 | freeaddrinfo(ainfo_save); | |
0b014ea6 | 893 | return -1; |
449cff3e | 894 | } |
d62a17ae | 895 | count = 0; |
896 | for (ainfo = ainfo_save; ainfo; ainfo = ainfo->ai_next) { | |
897 | int sock; | |
898 | ||
899 | if (ainfo->ai_family != AF_INET && ainfo->ai_family != AF_INET6) | |
900 | continue; | |
901 | ||
0cf6db21 | 902 | frr_with_privs(&bgpd_privs) { |
01b9e3fd DL |
903 | sock = vrf_socket(ainfo->ai_family, |
904 | ainfo->ai_socktype, | |
67684738 | 905 | ainfo->ai_protocol, |
2c1eba8e | 906 | bgp->vrf_id, |
633fc9b1 DL |
907 | (bgp->inst_type |
908 | == BGP_INSTANCE_TYPE_VRF | |
909 | ? bgp->name : NULL)); | |
01b9e3fd | 910 | } |
d62a17ae | 911 | if (sock < 0) { |
450971aa | 912 | flog_err_sys(EC_LIB_SOCKET, "socket: %s", |
09c866e3 | 913 | safe_strerror(errno)); |
d62a17ae | 914 | continue; |
915 | } | |
916 | ||
917 | /* if we intend to implement ttl-security, this socket needs | |
918 | * ttl=255 */ | |
919 | sockopt_ttl(ainfo->ai_family, sock, MAXTTL); | |
920 | ||
996c9314 LB |
921 | ret = bgp_listener(sock, ainfo->ai_addr, ainfo->ai_addrlen, |
922 | bgp); | |
d62a17ae | 923 | if (ret == 0) |
924 | ++count; | |
925 | else | |
926 | close(sock); | |
927 | } | |
928 | freeaddrinfo(ainfo_save); | |
06969768 | 929 | if (count == 0 && bgp->inst_type != BGP_INSTANCE_TYPE_VRF) { |
af4c2728 | 930 | flog_err( |
450971aa | 931 | EC_LIB_SOCKET, |
996c9314 LB |
932 | "%s: no usable addresses please check other programs usage of specified port %d", |
933 | __func__, port); | |
450971aa | 934 | flog_err_sys(EC_LIB_SOCKET, "%s: Program cannot continue", |
09c866e3 | 935 | __func__); |
b3b78f6e | 936 | exit(-1); |
d62a17ae | 937 | } |
938 | ||
939 | return 0; | |
718e3744 | 940 | } |
d023aec4 | 941 | |
e5619c28 PG |
942 | /* this function closes vrf socket |
943 | * this should be called only for vrf socket with netns backend | |
944 | */ | |
945 | void bgp_close_vrf_socket(struct bgp *bgp) | |
946 | { | |
947 | struct listnode *node, *next; | |
948 | struct bgp_listener *listener; | |
949 | ||
950 | if (!bgp) | |
951 | return; | |
952 | ||
953 | if (bm->listen_sockets == NULL) | |
954 | return; | |
955 | ||
956 | for (ALL_LIST_ELEMENTS(bm->listen_sockets, node, next, listener)) { | |
957 | if (listener->bgp == bgp) { | |
2f9db11b | 958 | THREAD_OFF(listener->thread); |
e5619c28 PG |
959 | close(listener->fd); |
960 | listnode_delete(bm->listen_sockets, listener); | |
2a0e69ae | 961 | XFREE(MTYPE_BGP_LISTENER, listener->name); |
e5619c28 PG |
962 | XFREE(MTYPE_BGP_LISTENER, listener); |
963 | } | |
964 | } | |
965 | } | |
966 | ||
967 | /* this function closes main socket | |
968 | */ | |
d62a17ae | 969 | void bgp_close(void) |
d023aec4 | 970 | { |
d62a17ae | 971 | struct listnode *node, *next; |
972 | struct bgp_listener *listener; | |
973 | ||
974 | if (bm->listen_sockets == NULL) | |
975 | return; | |
976 | ||
977 | for (ALL_LIST_ELEMENTS(bm->listen_sockets, node, next, listener)) { | |
e5619c28 PG |
978 | if (listener->bgp) |
979 | continue; | |
e2d409a8 | 980 | THREAD_OFF(listener->thread); |
d62a17ae | 981 | close(listener->fd); |
982 | listnode_delete(bm->listen_sockets, listener); | |
2a0e69ae | 983 | XFREE(MTYPE_BGP_LISTENER, listener->name); |
d62a17ae | 984 | XFREE(MTYPE_BGP_LISTENER, listener); |
985 | } | |
d023aec4 | 986 | } |