]> git.proxmox.com Git - mirror_frr.git/blob - bfdd/bfd_packet.c
Merge pull request #7368 from eololab/add-pidfile-in-frr.service
[mirror_frr.git] / bfdd / bfd_packet.c
1 /*********************************************************************
2 * Copyright 2017 Cumulus Networks, Inc. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along
15 * with this program; see the file COPYING; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 *
18 * bfd_packet.c: implements the BFD protocol packet handling.
19 *
20 * Authors
21 * -------
22 * Shrijeet Mukherjee [shm@cumulusnetworks.com]
23 * Kanna Rajagopal [kanna@cumulusnetworks.com]
24 * Radhika Mahankali [Radhika@cumulusnetworks.com]
25 */
26
27 #include <zebra.h>
28
29 #ifdef BFD_LINUX
30 #include <linux/if_packet.h>
31 #endif /* BFD_LINUX */
32
33 #include <netinet/if_ether.h>
34 #include <netinet/udp.h>
35
36 #include "lib/sockopt.h"
37
38 #include "bfd.h"
39
40 /*
41 * Prototypes
42 */
43 static int ptm_bfd_process_echo_pkt(struct bfd_vrf_global *bvrf, int s);
44 int _ptm_bfd_send(struct bfd_session *bs, uint16_t *port, const void *data,
45 size_t datalen);
46
47 static void bfd_sd_reschedule(struct bfd_vrf_global *bvrf, int sd);
48 ssize_t bfd_recv_ipv4(int sd, uint8_t *msgbuf, size_t msgbuflen, uint8_t *ttl,
49 ifindex_t *ifindex, struct sockaddr_any *local,
50 struct sockaddr_any *peer);
51 ssize_t bfd_recv_ipv6(int sd, uint8_t *msgbuf, size_t msgbuflen, uint8_t *ttl,
52 ifindex_t *ifindex, struct sockaddr_any *local,
53 struct sockaddr_any *peer);
54 int bp_udp_send(int sd, uint8_t ttl, uint8_t *data, size_t datalen,
55 struct sockaddr *to, socklen_t tolen);
56 int bp_bfd_echo_in(struct bfd_vrf_global *bvrf, int sd,
57 uint8_t *ttl, uint32_t *my_discr);
58
59 /* socket related prototypes */
60 static void bp_set_ipopts(int sd);
61 static void bp_bind_ip(int sd, uint16_t port);
62 static void bp_set_ipv6opts(int sd);
63 static void bp_bind_ipv6(int sd, uint16_t port);
64
65
66 /*
67 * Functions
68 */
69 int _ptm_bfd_send(struct bfd_session *bs, uint16_t *port, const void *data,
70 size_t datalen)
71 {
72 struct sockaddr *sa;
73 struct sockaddr_in sin;
74 struct sockaddr_in6 sin6;
75 socklen_t slen;
76 ssize_t rv;
77 int sd = -1;
78
79 if (CHECK_FLAG(bs->flags, BFD_SESS_FLAG_IPV6)) {
80 memset(&sin6, 0, sizeof(sin6));
81 sin6.sin6_family = AF_INET6;
82 memcpy(&sin6.sin6_addr, &bs->key.peer, sizeof(sin6.sin6_addr));
83 if (bs->ifp && IN6_IS_ADDR_LINKLOCAL(&sin6.sin6_addr))
84 sin6.sin6_scope_id = bs->ifp->ifindex;
85
86 sin6.sin6_port =
87 (port) ? *port
88 : (CHECK_FLAG(bs->flags, BFD_SESS_FLAG_MH))
89 ? htons(BFD_DEF_MHOP_DEST_PORT)
90 : htons(BFD_DEFDESTPORT);
91
92 sd = bs->sock;
93 sa = (struct sockaddr *)&sin6;
94 slen = sizeof(sin6);
95 } else {
96 memset(&sin, 0, sizeof(sin));
97 sin.sin_family = AF_INET;
98 memcpy(&sin.sin_addr, &bs->key.peer, sizeof(sin.sin_addr));
99 sin.sin_port =
100 (port) ? *port
101 : (CHECK_FLAG(bs->flags, BFD_SESS_FLAG_MH))
102 ? htons(BFD_DEF_MHOP_DEST_PORT)
103 : htons(BFD_DEFDESTPORT);
104
105 sd = bs->sock;
106 sa = (struct sockaddr *)&sin;
107 slen = sizeof(sin);
108 }
109
110 #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
111 sa->sa_len = slen;
112 #endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */
113 rv = sendto(sd, data, datalen, 0, sa, slen);
114 if (rv <= 0) {
115 if (bglobal.debug_network)
116 zlog_debug("packet-send: send failure: %s",
117 strerror(errno));
118 return -1;
119 }
120 if (rv < (ssize_t)datalen) {
121 if (bglobal.debug_network)
122 zlog_debug("packet-send: send partial: %s",
123 strerror(errno));
124 }
125
126 return 0;
127 }
128
129 void ptm_bfd_echo_snd(struct bfd_session *bfd)
130 {
131 struct sockaddr *sa;
132 socklen_t salen;
133 int sd;
134 struct bfd_echo_pkt bep;
135 struct sockaddr_in sin;
136 struct sockaddr_in6 sin6;
137 struct bfd_vrf_global *bvrf = bfd_vrf_look_by_session(bfd);
138
139 if (!bvrf)
140 return;
141 if (!CHECK_FLAG(bfd->flags, BFD_SESS_FLAG_ECHO_ACTIVE))
142 SET_FLAG(bfd->flags, BFD_SESS_FLAG_ECHO_ACTIVE);
143
144 memset(&bep, 0, sizeof(bep));
145 bep.ver = BFD_ECHO_VERSION;
146 bep.len = BFD_ECHO_PKT_LEN;
147 bep.my_discr = htonl(bfd->discrs.my_discr);
148
149 if (CHECK_FLAG(bfd->flags, BFD_SESS_FLAG_IPV6)) {
150 if (bvrf->bg_echov6 == -1)
151 return;
152 sd = bvrf->bg_echov6;
153 memset(&sin6, 0, sizeof(sin6));
154 sin6.sin6_family = AF_INET6;
155 memcpy(&sin6.sin6_addr, &bfd->key.peer, sizeof(sin6.sin6_addr));
156 if (bfd->ifp && IN6_IS_ADDR_LINKLOCAL(&sin6.sin6_addr))
157 sin6.sin6_scope_id = bfd->ifp->ifindex;
158
159 sin6.sin6_port = htons(BFD_DEF_ECHO_PORT);
160 #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
161 sin6.sin6_len = sizeof(sin6);
162 #endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */
163
164 sa = (struct sockaddr *)&sin6;
165 salen = sizeof(sin6);
166 } else {
167 sd = bvrf->bg_echo;
168 memset(&sin6, 0, sizeof(sin6));
169 sin.sin_family = AF_INET;
170 memcpy(&sin.sin_addr, &bfd->key.peer, sizeof(sin.sin_addr));
171 sin.sin_port = htons(BFD_DEF_ECHO_PORT);
172 #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
173 sin.sin_len = sizeof(sin);
174 #endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */
175
176 sa = (struct sockaddr *)&sin;
177 salen = sizeof(sin);
178 }
179 if (bp_udp_send(sd, BFD_TTL_VAL, (uint8_t *)&bep, sizeof(bep), sa,
180 salen)
181 == -1)
182 return;
183
184 bfd->stats.tx_echo_pkt++;
185 }
186
187 static int ptm_bfd_process_echo_pkt(struct bfd_vrf_global *bvrf, int s)
188 {
189 struct bfd_session *bfd;
190 uint32_t my_discr = 0;
191 uint8_t ttl = 0;
192
193 /* Receive and parse echo packet. */
194 if (bp_bfd_echo_in(bvrf, s, &ttl, &my_discr) == -1)
195 return 0;
196
197 /* Your discriminator not zero - use it to find session */
198 bfd = bfd_id_lookup(my_discr);
199 if (bfd == NULL) {
200 if (bglobal.debug_network)
201 zlog_debug("echo-packet: no matching session (id:%u)",
202 my_discr);
203 return -1;
204 }
205
206 if (!CHECK_FLAG(bfd->flags, BFD_SESS_FLAG_ECHO_ACTIVE)) {
207 if (bglobal.debug_network)
208 zlog_debug("echo-packet: echo disabled [%s] (id:%u)",
209 bs_to_string(bfd), my_discr);
210 return -1;
211 }
212
213 bfd->stats.rx_echo_pkt++;
214
215 /* Compute detect time */
216 bfd->echo_detect_TO = bfd->remote_detect_mult * bfd->echo_xmt_TO;
217
218 /* Update echo receive timeout. */
219 if (bfd->echo_detect_TO > 0)
220 bfd_echo_recvtimer_update(bfd);
221
222 return 0;
223 }
224
225 void ptm_bfd_snd(struct bfd_session *bfd, int fbit)
226 {
227 struct bfd_pkt cp = {};
228
229 /* Set fields according to section 6.5.7 */
230 cp.diag = bfd->local_diag;
231 BFD_SETVER(cp.diag, BFD_VERSION);
232 cp.flags = 0;
233 BFD_SETSTATE(cp.flags, bfd->ses_state);
234
235 if (CHECK_FLAG(bfd->flags, BFD_SESS_FLAG_CBIT))
236 BFD_SETCBIT(cp.flags, BFD_CBIT);
237
238 BFD_SETDEMANDBIT(cp.flags, BFD_DEF_DEMAND);
239
240 /*
241 * Polling and Final can't be set at the same time.
242 *
243 * RFC 5880, Section 6.5.
244 */
245 BFD_SETFBIT(cp.flags, fbit);
246 if (fbit == 0)
247 BFD_SETPBIT(cp.flags, bfd->polling);
248
249 cp.detect_mult = bfd->detect_mult;
250 cp.len = BFD_PKT_LEN;
251 cp.discrs.my_discr = htonl(bfd->discrs.my_discr);
252 cp.discrs.remote_discr = htonl(bfd->discrs.remote_discr);
253 if (bfd->polling) {
254 cp.timers.desired_min_tx =
255 htonl(bfd->timers.desired_min_tx);
256 cp.timers.required_min_rx =
257 htonl(bfd->timers.required_min_rx);
258 } else {
259 /*
260 * We can only announce current setting on poll, this
261 * avoids timing mismatch with our peer and give it
262 * the oportunity to learn. See `bs_final_handler` for
263 * more information.
264 */
265 cp.timers.desired_min_tx =
266 htonl(bfd->cur_timers.desired_min_tx);
267 cp.timers.required_min_rx =
268 htonl(bfd->cur_timers.required_min_rx);
269 }
270 cp.timers.required_min_echo = htonl(bfd->timers.required_min_echo);
271
272 if (_ptm_bfd_send(bfd, NULL, &cp, BFD_PKT_LEN) != 0)
273 return;
274
275 bfd->stats.tx_ctrl_pkt++;
276 }
277
278 ssize_t bfd_recv_ipv4(int sd, uint8_t *msgbuf, size_t msgbuflen, uint8_t *ttl,
279 ifindex_t *ifindex, struct sockaddr_any *local,
280 struct sockaddr_any *peer)
281 {
282 struct cmsghdr *cm;
283 ssize_t mlen;
284 struct sockaddr_in msgaddr;
285 struct msghdr msghdr;
286 struct iovec iov[1];
287 uint8_t cmsgbuf[255];
288
289 /* Prepare the recvmsg params. */
290 iov[0].iov_base = msgbuf;
291 iov[0].iov_len = msgbuflen;
292
293 memset(&msghdr, 0, sizeof(msghdr));
294 msghdr.msg_name = &msgaddr;
295 msghdr.msg_namelen = sizeof(msgaddr);
296 msghdr.msg_iov = iov;
297 msghdr.msg_iovlen = 1;
298 msghdr.msg_control = cmsgbuf;
299 msghdr.msg_controllen = sizeof(cmsgbuf);
300
301 mlen = recvmsg(sd, &msghdr, MSG_DONTWAIT);
302 if (mlen == -1) {
303 if (errno != EAGAIN)
304 zlog_err("ipv4-recv: recv failed: %s", strerror(errno));
305
306 return -1;
307 }
308
309 /* Get source address */
310 peer->sa_sin = *((struct sockaddr_in *)(msghdr.msg_name));
311
312 /* Get and check TTL */
313 for (cm = CMSG_FIRSTHDR(&msghdr); cm != NULL;
314 cm = CMSG_NXTHDR(&msghdr, cm)) {
315 if (cm->cmsg_level != IPPROTO_IP)
316 continue;
317
318 switch (cm->cmsg_type) {
319 #ifdef BFD_LINUX
320 case IP_TTL: {
321 uint32_t ttlval;
322
323 memcpy(&ttlval, CMSG_DATA(cm), sizeof(ttlval));
324 if (ttlval > 255) {
325 if (bglobal.debug_network)
326 zlog_debug("ipv4-recv: invalid TTL: %u",
327 ttlval);
328 return -1;
329 }
330 *ttl = ttlval;
331 break;
332 }
333
334 case IP_PKTINFO: {
335 struct in_pktinfo *pi =
336 (struct in_pktinfo *)CMSG_DATA(cm);
337
338 if (pi == NULL)
339 break;
340
341 local->sa_sin.sin_family = AF_INET;
342 local->sa_sin.sin_addr = pi->ipi_addr;
343 #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
344 local->sa_sin.sin_len = sizeof(local->sa_sin);
345 #endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */
346
347 *ifindex = pi->ipi_ifindex;
348 break;
349 }
350 #endif /* BFD_LINUX */
351 #ifdef BFD_BSD
352 case IP_RECVTTL: {
353 memcpy(ttl, CMSG_DATA(cm), sizeof(*ttl));
354 break;
355 }
356
357 case IP_RECVDSTADDR: {
358 struct in_addr ia;
359
360 memcpy(&ia, CMSG_DATA(cm), sizeof(ia));
361 local->sa_sin.sin_family = AF_INET;
362 local->sa_sin.sin_addr = ia;
363 #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
364 local->sa_sin.sin_len = sizeof(local->sa_sin);
365 #endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */
366 break;
367 }
368 #endif /* BFD_BSD */
369
370 default:
371 /*
372 * On *BSDs we expect to land here when skipping
373 * the IP_RECVIF header. It will be handled by
374 * getsockopt_ifindex() below.
375 */
376 /* NOTHING */
377 break;
378 }
379 }
380
381 /* OS agnostic way of getting interface name. */
382 if (*ifindex == IFINDEX_INTERNAL)
383 *ifindex = getsockopt_ifindex(AF_INET, &msghdr);
384
385 return mlen;
386 }
387
388 ssize_t bfd_recv_ipv6(int sd, uint8_t *msgbuf, size_t msgbuflen, uint8_t *ttl,
389 ifindex_t *ifindex, struct sockaddr_any *local,
390 struct sockaddr_any *peer)
391 {
392 struct cmsghdr *cm;
393 struct in6_pktinfo *pi6 = NULL;
394 ssize_t mlen;
395 uint32_t ttlval;
396 struct sockaddr_in6 msgaddr6;
397 struct msghdr msghdr6;
398 struct iovec iov[1];
399 uint8_t cmsgbuf6[255];
400
401 /* Prepare the recvmsg params. */
402 iov[0].iov_base = msgbuf;
403 iov[0].iov_len = msgbuflen;
404
405 memset(&msghdr6, 0, sizeof(msghdr6));
406 msghdr6.msg_name = &msgaddr6;
407 msghdr6.msg_namelen = sizeof(msgaddr6);
408 msghdr6.msg_iov = iov;
409 msghdr6.msg_iovlen = 1;
410 msghdr6.msg_control = cmsgbuf6;
411 msghdr6.msg_controllen = sizeof(cmsgbuf6);
412
413 mlen = recvmsg(sd, &msghdr6, MSG_DONTWAIT);
414 if (mlen == -1) {
415 if (errno != EAGAIN)
416 zlog_err("ipv6-recv: recv failed: %s", strerror(errno));
417
418 return -1;
419 }
420
421 /* Get source address */
422 peer->sa_sin6 = *((struct sockaddr_in6 *)(msghdr6.msg_name));
423
424 /* Get and check TTL */
425 for (cm = CMSG_FIRSTHDR(&msghdr6); cm != NULL;
426 cm = CMSG_NXTHDR(&msghdr6, cm)) {
427 if (cm->cmsg_level != IPPROTO_IPV6)
428 continue;
429
430 if (cm->cmsg_type == IPV6_HOPLIMIT) {
431 memcpy(&ttlval, CMSG_DATA(cm), sizeof(ttlval));
432 if (ttlval > 255) {
433 if (bglobal.debug_network)
434 zlog_debug("ipv6-recv: invalid TTL: %u",
435 ttlval);
436 return -1;
437 }
438
439 *ttl = ttlval;
440 } else if (cm->cmsg_type == IPV6_PKTINFO) {
441 pi6 = (struct in6_pktinfo *)CMSG_DATA(cm);
442 if (pi6) {
443 local->sa_sin6.sin6_family = AF_INET6;
444 local->sa_sin6.sin6_addr = pi6->ipi6_addr;
445 #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
446 local->sa_sin6.sin6_len = sizeof(local->sa_sin6);
447 #endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */
448
449 *ifindex = pi6->ipi6_ifindex;
450
451 /* Set scope ID for link local addresses. */
452 if (IN6_IS_ADDR_LINKLOCAL(
453 &peer->sa_sin6.sin6_addr))
454 peer->sa_sin6.sin6_scope_id = *ifindex;
455 if (IN6_IS_ADDR_LINKLOCAL(
456 &local->sa_sin6.sin6_addr))
457 local->sa_sin6.sin6_scope_id = *ifindex;
458 }
459 }
460 }
461
462 return mlen;
463 }
464
465 static void bfd_sd_reschedule(struct bfd_vrf_global *bvrf, int sd)
466 {
467 if (sd == bvrf->bg_shop) {
468 THREAD_OFF(bvrf->bg_ev[0]);
469 thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_shop,
470 &bvrf->bg_ev[0]);
471 } else if (sd == bvrf->bg_mhop) {
472 THREAD_OFF(bvrf->bg_ev[1]);
473 thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_mhop,
474 &bvrf->bg_ev[1]);
475 } else if (sd == bvrf->bg_shop6) {
476 THREAD_OFF(bvrf->bg_ev[2]);
477 thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_shop6,
478 &bvrf->bg_ev[2]);
479 } else if (sd == bvrf->bg_mhop6) {
480 THREAD_OFF(bvrf->bg_ev[3]);
481 thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_mhop6,
482 &bvrf->bg_ev[3]);
483 } else if (sd == bvrf->bg_echo) {
484 THREAD_OFF(bvrf->bg_ev[4]);
485 thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_echo,
486 &bvrf->bg_ev[4]);
487 } else if (sd == bvrf->bg_echov6) {
488 THREAD_OFF(bvrf->bg_ev[5]);
489 thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_echov6,
490 &bvrf->bg_ev[5]);
491 }
492 }
493
494 static void cp_debug(bool mhop, struct sockaddr_any *peer,
495 struct sockaddr_any *local, ifindex_t ifindex,
496 vrf_id_t vrfid, const char *fmt, ...)
497 {
498 char buf[512], peerstr[128], localstr[128], portstr[64], vrfstr[64];
499 va_list vl;
500
501 /* Don't to any processing if debug is disabled. */
502 if (bglobal.debug_network == false)
503 return;
504
505 if (peer->sa_sin.sin_family)
506 snprintf(peerstr, sizeof(peerstr), " peer:%s", satostr(peer));
507 else
508 peerstr[0] = 0;
509
510 if (local->sa_sin.sin_family)
511 snprintf(localstr, sizeof(localstr), " local:%s",
512 satostr(local));
513 else
514 localstr[0] = 0;
515
516 if (ifindex != IFINDEX_INTERNAL)
517 snprintf(portstr, sizeof(portstr), " port:%u", ifindex);
518 else
519 portstr[0] = 0;
520
521 if (vrfid != VRF_DEFAULT)
522 snprintf(vrfstr, sizeof(vrfstr), " vrf:%u", vrfid);
523 else
524 vrfstr[0] = 0;
525
526 va_start(vl, fmt);
527 vsnprintf(buf, sizeof(buf), fmt, vl);
528 va_end(vl);
529
530 zlog_debug("control-packet: %s [mhop:%s%s%s%s%s]", buf,
531 mhop ? "yes" : "no", peerstr, localstr, portstr, vrfstr);
532 }
533
534 int bfd_recv_cb(struct thread *t)
535 {
536 int sd = THREAD_FD(t);
537 struct bfd_session *bfd;
538 struct bfd_pkt *cp;
539 bool is_mhop;
540 ssize_t mlen = 0;
541 uint8_t ttl = 0;
542 vrf_id_t vrfid;
543 ifindex_t ifindex = IFINDEX_INTERNAL;
544 struct sockaddr_any local, peer;
545 uint8_t msgbuf[1516];
546 struct interface *ifp = NULL;
547 struct bfd_vrf_global *bvrf = THREAD_ARG(t);
548
549 vrfid = bvrf->vrf->vrf_id;
550
551 /* Schedule next read. */
552 bfd_sd_reschedule(bvrf, sd);
553
554 /* Handle echo packets. */
555 if (sd == bvrf->bg_echo || sd == bvrf->bg_echov6) {
556 ptm_bfd_process_echo_pkt(bvrf, sd);
557 return 0;
558 }
559
560 /* Sanitize input/output. */
561 memset(&local, 0, sizeof(local));
562 memset(&peer, 0, sizeof(peer));
563
564 /* Handle control packets. */
565 is_mhop = false;
566 if (sd == bvrf->bg_shop || sd == bvrf->bg_mhop) {
567 is_mhop = sd == bvrf->bg_mhop;
568 mlen = bfd_recv_ipv4(sd, msgbuf, sizeof(msgbuf), &ttl, &ifindex,
569 &local, &peer);
570 } else if (sd == bvrf->bg_shop6 || sd == bvrf->bg_mhop6) {
571 is_mhop = sd == bvrf->bg_mhop6;
572 mlen = bfd_recv_ipv6(sd, msgbuf, sizeof(msgbuf), &ttl, &ifindex,
573 &local, &peer);
574 }
575
576 /* update vrf-id because when in vrf-lite mode,
577 * the socket is on default namespace
578 */
579 if (ifindex) {
580 ifp = if_lookup_by_index(ifindex, vrfid);
581 if (ifp)
582 vrfid = ifp->vrf_id;
583 }
584
585 /* Implement RFC 5880 6.8.6 */
586 if (mlen < BFD_PKT_LEN) {
587 cp_debug(is_mhop, &peer, &local, ifindex, vrfid,
588 "too small (%ld bytes)", mlen);
589 return 0;
590 }
591
592 /* Validate single hop packet TTL. */
593 if ((!is_mhop) && (ttl != BFD_TTL_VAL)) {
594 cp_debug(is_mhop, &peer, &local, ifindex, vrfid,
595 "invalid TTL: %d expected %d", ttl, BFD_TTL_VAL);
596 return 0;
597 }
598
599 /*
600 * Parse the control header for inconsistencies:
601 * - Invalid version;
602 * - Bad multiplier configuration;
603 * - Short packets;
604 * - Invalid discriminator;
605 */
606 cp = (struct bfd_pkt *)(msgbuf);
607 if (BFD_GETVER(cp->diag) != BFD_VERSION) {
608 cp_debug(is_mhop, &peer, &local, ifindex, vrfid,
609 "bad version %d", BFD_GETVER(cp->diag));
610 return 0;
611 }
612
613 if (cp->detect_mult == 0) {
614 cp_debug(is_mhop, &peer, &local, ifindex, vrfid,
615 "detect multiplier set to zero");
616 return 0;
617 }
618
619 if ((cp->len < BFD_PKT_LEN) || (cp->len > mlen)) {
620 cp_debug(is_mhop, &peer, &local, ifindex, vrfid, "too small");
621 return 0;
622 }
623
624 if (cp->discrs.my_discr == 0) {
625 cp_debug(is_mhop, &peer, &local, ifindex, vrfid,
626 "'my discriminator' is zero");
627 return 0;
628 }
629
630 /* Find the session that this packet belongs. */
631 bfd = ptm_bfd_sess_find(cp, &peer, &local, ifindex, vrfid, is_mhop);
632 if (bfd == NULL) {
633 cp_debug(is_mhop, &peer, &local, ifindex, vrfid,
634 "no session found");
635 return 0;
636 }
637
638 bfd->stats.rx_ctrl_pkt++;
639
640 /*
641 * Multi hop: validate packet TTL.
642 * Single hop: set local address that received the packet.
643 */
644 if (is_mhop) {
645 if (ttl < bfd->mh_ttl) {
646 cp_debug(is_mhop, &peer, &local, ifindex, vrfid,
647 "exceeded max hop count (expected %d, got %d)",
648 bfd->mh_ttl, ttl);
649 return 0;
650 }
651 } else if (bfd->local_address.sa_sin.sin_family == AF_UNSPEC) {
652 bfd->local_address = local;
653 }
654
655 /*
656 * If no interface was detected, save the interface where the
657 * packet came in.
658 */
659 if (bfd->ifp == NULL)
660 bfd->ifp = if_lookup_by_index(ifindex, vrfid);
661
662 /* Log remote discriminator changes. */
663 if ((bfd->discrs.remote_discr != 0)
664 && (bfd->discrs.remote_discr != ntohl(cp->discrs.my_discr)))
665 cp_debug(is_mhop, &peer, &local, ifindex, vrfid,
666 "remote discriminator mismatch (expected %u, got %u)",
667 bfd->discrs.remote_discr, ntohl(cp->discrs.my_discr));
668
669 bfd->discrs.remote_discr = ntohl(cp->discrs.my_discr);
670
671 /* Save remote diagnostics before state switch. */
672 bfd->remote_diag = cp->diag & BFD_DIAGMASK;
673
674 /* Update remote timers settings. */
675 bfd->remote_timers.desired_min_tx = ntohl(cp->timers.desired_min_tx);
676 bfd->remote_timers.required_min_rx = ntohl(cp->timers.required_min_rx);
677 bfd->remote_timers.required_min_echo =
678 ntohl(cp->timers.required_min_echo);
679 bfd->remote_detect_mult = cp->detect_mult;
680
681 if (BFD_GETCBIT(cp->flags))
682 bfd->remote_cbit = 1;
683 else
684 bfd->remote_cbit = 0;
685
686 /* State switch from section 6.2. */
687 bs_state_handler(bfd, BFD_GETSTATE(cp->flags));
688
689 /* RFC 5880, Section 6.5: handle POLL/FINAL negotiation sequence. */
690 if (bfd->polling && BFD_GETFBIT(cp->flags)) {
691 /* Disable pooling. */
692 bfd->polling = 0;
693
694 /* Handle poll finalization. */
695 bs_final_handler(bfd);
696 } else {
697 /* Received a packet, lets update the receive timer. */
698 bfd_recvtimer_update(bfd);
699 }
700
701 /* Handle echo timers changes. */
702 bs_echo_timer_handler(bfd);
703
704 /*
705 * We've received a packet with the POLL bit set, we must send
706 * a control packet back with the FINAL bit set.
707 *
708 * RFC 5880, Section 6.5.
709 */
710 if (BFD_GETPBIT(cp->flags)) {
711 /* We are finalizing a poll negotiation. */
712 bs_final_handler(bfd);
713
714 /* Send the control packet with the final bit immediately. */
715 ptm_bfd_snd(bfd, 1);
716 }
717
718 return 0;
719 }
720
721 /*
722 * bp_bfd_echo_in: proccesses an BFD echo packet. On TTL == BFD_TTL_VAL
723 * the packet is looped back or returns the my discriminator ID along
724 * with the TTL.
725 *
726 * Returns -1 on error or loopback or 0 on success.
727 */
728 int bp_bfd_echo_in(struct bfd_vrf_global *bvrf, int sd,
729 uint8_t *ttl, uint32_t *my_discr)
730 {
731 struct bfd_echo_pkt *bep;
732 ssize_t rlen;
733 struct sockaddr_any local, peer;
734 ifindex_t ifindex = IFINDEX_INTERNAL;
735 vrf_id_t vrfid = VRF_DEFAULT;
736 uint8_t msgbuf[1516];
737
738 if (sd == bvrf->bg_echo)
739 rlen = bfd_recv_ipv4(sd, msgbuf, sizeof(msgbuf), ttl, &ifindex,
740 &local, &peer);
741 else
742 rlen = bfd_recv_ipv6(sd, msgbuf, sizeof(msgbuf), ttl, &ifindex,
743 &local, &peer);
744
745 /* Short packet, better not risk reading it. */
746 if (rlen < (ssize_t)sizeof(*bep)) {
747 cp_debug(false, &peer, &local, ifindex, vrfid,
748 "small echo packet");
749 return -1;
750 }
751
752 /* Test for loopback. */
753 if (*ttl == BFD_TTL_VAL) {
754 bp_udp_send(sd, *ttl - 1, msgbuf, rlen,
755 (struct sockaddr *)&peer,
756 (sd == bvrf->bg_echo) ? sizeof(peer.sa_sin)
757 : sizeof(peer.sa_sin6));
758 return -1;
759 }
760
761 /* Read my discriminator from BFD Echo packet. */
762 bep = (struct bfd_echo_pkt *)msgbuf;
763 *my_discr = ntohl(bep->my_discr);
764 if (*my_discr == 0) {
765 cp_debug(false, &peer, &local, ifindex, vrfid,
766 "invalid echo packet discriminator (zero)");
767 return -1;
768 }
769
770 return 0;
771 }
772
773 int bp_udp_send(int sd, uint8_t ttl, uint8_t *data, size_t datalen,
774 struct sockaddr *to, socklen_t tolen)
775 {
776 struct cmsghdr *cmsg;
777 ssize_t wlen;
778 int ttlval = ttl;
779 bool is_ipv6 = to->sa_family == AF_INET6;
780 struct msghdr msg;
781 struct iovec iov[1];
782 uint8_t msgctl[255];
783
784 /* Prepare message data. */
785 iov[0].iov_base = data;
786 iov[0].iov_len = datalen;
787
788 memset(&msg, 0, sizeof(msg));
789 memset(msgctl, 0, sizeof(msgctl));
790 msg.msg_name = to;
791 msg.msg_namelen = tolen;
792 msg.msg_iov = iov;
793 msg.msg_iovlen = 1;
794
795 /* Prepare the packet TTL information. */
796 if (ttl > 0) {
797 /* Use ancillary data. */
798 msg.msg_control = msgctl;
799 msg.msg_controllen = CMSG_LEN(sizeof(ttlval));
800
801 /* Configure the ancillary data. */
802 cmsg = CMSG_FIRSTHDR(&msg);
803 cmsg->cmsg_len = CMSG_LEN(sizeof(ttlval));
804 if (is_ipv6) {
805 cmsg->cmsg_level = IPPROTO_IPV6;
806 cmsg->cmsg_type = IPV6_HOPLIMIT;
807 } else {
808 #ifdef BFD_LINUX
809 cmsg->cmsg_level = IPPROTO_IP;
810 cmsg->cmsg_type = IP_TTL;
811 #else
812 /* FreeBSD does not support TTL in ancillary data. */
813 msg.msg_control = NULL;
814 msg.msg_controllen = 0;
815
816 bp_set_ttl(sd, ttl);
817 #endif /* BFD_BSD */
818 }
819 memcpy(CMSG_DATA(cmsg), &ttlval, sizeof(ttlval));
820 }
821
822 /* Send echo back. */
823 wlen = sendmsg(sd, &msg, 0);
824 if (wlen <= 0) {
825 if (bglobal.debug_network)
826 zlog_debug("udp-send: loopback failure: (%d) %s", errno,
827 strerror(errno));
828 return -1;
829 } else if (wlen < (ssize_t)datalen) {
830 if (bglobal.debug_network)
831 zlog_debug("udp-send: partial send: %zd expected %zu",
832 wlen, datalen);
833 return -1;
834 }
835
836 return 0;
837 }
838
839
840 /*
841 * Sockets creation.
842 */
843
844
845 /*
846 * IPv4 sockets
847 */
848 int bp_set_ttl(int sd, uint8_t value)
849 {
850 int ttl = value;
851
852 if (setsockopt(sd, IPPROTO_IP, IP_TTL, &ttl, sizeof(ttl)) == -1) {
853 zlog_warn("set-ttl: setsockopt(IP_TTL, %d): %s", value,
854 strerror(errno));
855 return -1;
856 }
857
858 return 0;
859 }
860
861 int bp_set_tos(int sd, uint8_t value)
862 {
863 int tos = value;
864
865 if (setsockopt(sd, IPPROTO_IP, IP_TOS, &tos, sizeof(tos)) == -1) {
866 zlog_warn("set-tos: setsockopt(IP_TOS, %d): %s", value,
867 strerror(errno));
868 return -1;
869 }
870
871 return 0;
872 }
873
874 static void bp_set_ipopts(int sd)
875 {
876 int rcvttl = BFD_RCV_TTL_VAL;
877
878 if (bp_set_ttl(sd, BFD_TTL_VAL) != 0)
879 zlog_fatal("set-ipopts: TTL configuration failed");
880
881 if (setsockopt(sd, IPPROTO_IP, IP_RECVTTL, &rcvttl, sizeof(rcvttl))
882 == -1)
883 zlog_fatal("set-ipopts: setsockopt(IP_RECVTTL, %d): %s", rcvttl,
884 strerror(errno));
885
886 #ifdef BFD_LINUX
887 int pktinfo = BFD_PKT_INFO_VAL;
888
889 /* Figure out address and interface to do the peer matching. */
890 if (setsockopt(sd, IPPROTO_IP, IP_PKTINFO, &pktinfo, sizeof(pktinfo))
891 == -1)
892 zlog_fatal("set-ipopts: setsockopt(IP_PKTINFO, %d): %s",
893 pktinfo, strerror(errno));
894 #endif /* BFD_LINUX */
895 #ifdef BFD_BSD
896 int yes = 1;
897
898 /* Find out our address for peer matching. */
899 if (setsockopt(sd, IPPROTO_IP, IP_RECVDSTADDR, &yes, sizeof(yes)) == -1)
900 zlog_fatal("set-ipopts: setsockopt(IP_RECVDSTADDR, %d): %s",
901 yes, strerror(errno));
902
903 /* Find out interface where the packet came in. */
904 if (setsockopt_ifindex(AF_INET, sd, yes) == -1)
905 zlog_fatal("set-ipopts: setsockopt_ipv4_ifindex(%d): %s", yes,
906 strerror(errno));
907 #endif /* BFD_BSD */
908 }
909
910 static void bp_bind_ip(int sd, uint16_t port)
911 {
912 struct sockaddr_in sin;
913
914 memset(&sin, 0, sizeof(sin));
915 sin.sin_family = AF_INET;
916 sin.sin_addr.s_addr = htonl(INADDR_ANY);
917 sin.sin_port = htons(port);
918 if (bind(sd, (struct sockaddr *)&sin, sizeof(sin)) == -1)
919 zlog_fatal("bind-ip: bind: %s", strerror(errno));
920 }
921
922 int bp_udp_shop(const struct vrf *vrf)
923 {
924 int sd;
925
926 frr_with_privs(&bglobal.bfdd_privs) {
927 sd = vrf_socket(AF_INET, SOCK_DGRAM, PF_UNSPEC, vrf->vrf_id,
928 vrf->name);
929 }
930 if (sd == -1)
931 zlog_fatal("udp-shop: socket: %s", strerror(errno));
932
933 bp_set_ipopts(sd);
934 bp_bind_ip(sd, BFD_DEFDESTPORT);
935 return sd;
936 }
937
938 int bp_udp_mhop(const struct vrf *vrf)
939 {
940 int sd;
941
942 frr_with_privs(&bglobal.bfdd_privs) {
943 sd = vrf_socket(AF_INET, SOCK_DGRAM, PF_UNSPEC, vrf->vrf_id,
944 vrf->name);
945 }
946 if (sd == -1)
947 zlog_fatal("udp-mhop: socket: %s", strerror(errno));
948
949 bp_set_ipopts(sd);
950 bp_bind_ip(sd, BFD_DEF_MHOP_DEST_PORT);
951
952 return sd;
953 }
954
955 int bp_peer_socket(const struct bfd_session *bs)
956 {
957 int sd, pcount;
958 struct sockaddr_in sin;
959 static int srcPort = BFD_SRCPORTINIT;
960 const char *device_to_bind = NULL;
961
962 if (bs->key.ifname[0])
963 device_to_bind = (const char *)bs->key.ifname;
964 else if ((!vrf_is_backend_netns() && bs->vrf->vrf_id != VRF_DEFAULT)
965 || ((CHECK_FLAG(bs->flags, BFD_SESS_FLAG_MH)
966 && bs->key.vrfname[0])))
967 device_to_bind = (const char *)bs->key.vrfname;
968
969 frr_with_privs(&bglobal.bfdd_privs) {
970 sd = vrf_socket(AF_INET, SOCK_DGRAM, PF_UNSPEC,
971 bs->vrf->vrf_id, device_to_bind);
972 }
973 if (sd == -1) {
974 zlog_err("ipv4-new: failed to create socket: %s",
975 strerror(errno));
976 return -1;
977 }
978
979 /* Set TTL to 255 for all transmitted packets */
980 if (bp_set_ttl(sd, BFD_TTL_VAL) != 0) {
981 close(sd);
982 return -1;
983 }
984
985 /* Set TOS to CS6 for all transmitted packets */
986 if (bp_set_tos(sd, BFD_TOS_VAL) != 0) {
987 close(sd);
988 return -1;
989 }
990
991 /* Find an available source port in the proper range */
992 memset(&sin, 0, sizeof(sin));
993 sin.sin_family = AF_INET;
994 #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
995 sin.sin_len = sizeof(sin);
996 #endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */
997 memcpy(&sin.sin_addr, &bs->key.local, sizeof(sin.sin_addr));
998 if (CHECK_FLAG(bs->flags, BFD_SESS_FLAG_MH) == 0)
999 sin.sin_addr.s_addr = INADDR_ANY;
1000
1001 pcount = 0;
1002 do {
1003 if ((++pcount) > (BFD_SRCPORTMAX - BFD_SRCPORTINIT)) {
1004 /* Searched all ports, none available */
1005 zlog_err("ipv4-new: failed to bind port: %s",
1006 strerror(errno));
1007 close(sd);
1008 return -1;
1009 }
1010 if (srcPort >= BFD_SRCPORTMAX)
1011 srcPort = BFD_SRCPORTINIT;
1012 sin.sin_port = htons(srcPort++);
1013 } while (bind(sd, (struct sockaddr *)&sin, sizeof(sin)) < 0);
1014
1015 return sd;
1016 }
1017
1018
1019 /*
1020 * IPv6 sockets
1021 */
1022
1023 int bp_peer_socketv6(const struct bfd_session *bs)
1024 {
1025 int sd, pcount;
1026 struct sockaddr_in6 sin6;
1027 static int srcPort = BFD_SRCPORTINIT;
1028 const char *device_to_bind = NULL;
1029
1030 if (bs->key.ifname[0])
1031 device_to_bind = (const char *)bs->key.ifname;
1032 else if ((!vrf_is_backend_netns() && bs->vrf->vrf_id != VRF_DEFAULT)
1033 || ((CHECK_FLAG(bs->flags, BFD_SESS_FLAG_MH)
1034 && bs->key.vrfname[0])))
1035 device_to_bind = (const char *)bs->key.vrfname;
1036
1037 frr_with_privs(&bglobal.bfdd_privs) {
1038 sd = vrf_socket(AF_INET6, SOCK_DGRAM, PF_UNSPEC,
1039 bs->vrf->vrf_id, device_to_bind);
1040 }
1041 if (sd == -1) {
1042 zlog_err("ipv6-new: failed to create socket: %s",
1043 strerror(errno));
1044 return -1;
1045 }
1046
1047 /* Set TTL to 255 for all transmitted packets */
1048 if (bp_set_ttlv6(sd, BFD_TTL_VAL) != 0) {
1049 close(sd);
1050 return -1;
1051 }
1052
1053 /* Set TOS to CS6 for all transmitted packets */
1054 if (bp_set_tosv6(sd, BFD_TOS_VAL) != 0) {
1055 close(sd);
1056 return -1;
1057 }
1058
1059 /* Find an available source port in the proper range */
1060 memset(&sin6, 0, sizeof(sin6));
1061 sin6.sin6_family = AF_INET6;
1062 #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
1063 sin6.sin6_len = sizeof(sin6);
1064 #endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */
1065 memcpy(&sin6.sin6_addr, &bs->key.local, sizeof(sin6.sin6_addr));
1066 if (bs->ifp && IN6_IS_ADDR_LINKLOCAL(&sin6.sin6_addr))
1067 sin6.sin6_scope_id = bs->ifp->ifindex;
1068
1069 pcount = 0;
1070 do {
1071 if ((++pcount) > (BFD_SRCPORTMAX - BFD_SRCPORTINIT)) {
1072 /* Searched all ports, none available */
1073 zlog_err("ipv6-new: failed to bind port: %s",
1074 strerror(errno));
1075 close(sd);
1076 return -1;
1077 }
1078 if (srcPort >= BFD_SRCPORTMAX)
1079 srcPort = BFD_SRCPORTINIT;
1080 sin6.sin6_port = htons(srcPort++);
1081 } while (bind(sd, (struct sockaddr *)&sin6, sizeof(sin6)) < 0);
1082
1083 return sd;
1084 }
1085
1086 int bp_set_ttlv6(int sd, uint8_t value)
1087 {
1088 int ttl = value;
1089
1090 if (setsockopt(sd, IPPROTO_IPV6, IPV6_UNICAST_HOPS, &ttl, sizeof(ttl))
1091 == -1) {
1092 zlog_warn("set-ttlv6: setsockopt(IPV6_UNICAST_HOPS, %d): %s",
1093 value, strerror(errno));
1094 return -1;
1095 }
1096
1097 return 0;
1098 }
1099
1100 int bp_set_tosv6(int sd, uint8_t value)
1101 {
1102 int tos = value;
1103
1104 if (setsockopt(sd, IPPROTO_IPV6, IPV6_TCLASS, &tos, sizeof(tos))
1105 == -1) {
1106 zlog_warn("set-tosv6: setsockopt(IPV6_TCLASS, %d): %s", value,
1107 strerror(errno));
1108 return -1;
1109 }
1110
1111 return 0;
1112 }
1113
1114 static void bp_set_ipv6opts(int sd)
1115 {
1116 int ipv6_pktinfo = BFD_IPV6_PKT_INFO_VAL;
1117 int ipv6_only = BFD_IPV6_ONLY_VAL;
1118
1119 if (bp_set_ttlv6(sd, BFD_TTL_VAL) == -1)
1120 zlog_fatal(
1121 "set-ipv6opts: setsockopt(IPV6_UNICAST_HOPS, %d): %s",
1122 BFD_TTL_VAL, strerror(errno));
1123
1124 if (setsockopt_ipv6_hoplimit(sd, BFD_RCV_TTL_VAL) == -1)
1125 zlog_fatal("set-ipv6opts: setsockopt(IPV6_HOPLIMIT, %d): %s",
1126 BFD_RCV_TTL_VAL, strerror(errno));
1127
1128 if (setsockopt_ipv6_pktinfo(sd, ipv6_pktinfo) == -1)
1129 zlog_fatal("set-ipv6opts: setsockopt(IPV6_PKTINFO, %d): %s",
1130 ipv6_pktinfo, strerror(errno));
1131
1132 if (setsockopt(sd, IPPROTO_IPV6, IPV6_V6ONLY, &ipv6_only,
1133 sizeof(ipv6_only))
1134 == -1)
1135 zlog_fatal("set-ipv6opts: setsockopt(IPV6_V6ONLY, %d): %s",
1136 ipv6_only, strerror(errno));
1137 }
1138
1139 static void bp_bind_ipv6(int sd, uint16_t port)
1140 {
1141 struct sockaddr_in6 sin6;
1142
1143 memset(&sin6, 0, sizeof(sin6));
1144 sin6.sin6_family = AF_INET6;
1145 sin6.sin6_addr = in6addr_any;
1146 sin6.sin6_port = htons(port);
1147 #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
1148 sin6.sin6_len = sizeof(sin6);
1149 #endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */
1150 if (bind(sd, (struct sockaddr *)&sin6, sizeof(sin6)) == -1)
1151 zlog_fatal("bind-ipv6: bind: %s", strerror(errno));
1152 }
1153
1154 int bp_udp6_shop(const struct vrf *vrf)
1155 {
1156 int sd;
1157
1158 frr_with_privs(&bglobal.bfdd_privs) {
1159 sd = vrf_socket(AF_INET6, SOCK_DGRAM, PF_UNSPEC, vrf->vrf_id,
1160 vrf->name);
1161 }
1162 if (sd == -1) {
1163 if (errno != EAFNOSUPPORT)
1164 zlog_fatal("udp6-shop: socket: %s", strerror(errno));
1165 else
1166 zlog_warn("udp6-shop: V6 is not supported, continuing");
1167
1168 return -1;
1169 }
1170
1171 bp_set_ipv6opts(sd);
1172 bp_bind_ipv6(sd, BFD_DEFDESTPORT);
1173
1174 return sd;
1175 }
1176
1177 int bp_udp6_mhop(const struct vrf *vrf)
1178 {
1179 int sd;
1180
1181 frr_with_privs(&bglobal.bfdd_privs) {
1182 sd = vrf_socket(AF_INET6, SOCK_DGRAM, PF_UNSPEC, vrf->vrf_id,
1183 vrf->name);
1184 }
1185 if (sd == -1) {
1186 if (errno != EAFNOSUPPORT)
1187 zlog_fatal("udp6-mhop: socket: %s", strerror(errno));
1188 else
1189 zlog_warn("udp6-mhop: V6 is not supported, continuing");
1190
1191 return -1;
1192 }
1193
1194 bp_set_ipv6opts(sd);
1195 bp_bind_ipv6(sd, BFD_DEF_MHOP_DEST_PORT);
1196
1197 return sd;
1198 }
1199
1200 int bp_echo_socket(const struct vrf *vrf)
1201 {
1202 int s;
1203
1204 frr_with_privs(&bglobal.bfdd_privs) {
1205 s = vrf_socket(AF_INET, SOCK_DGRAM, 0, vrf->vrf_id, vrf->name);
1206 }
1207 if (s == -1)
1208 zlog_fatal("echo-socket: socket: %s", strerror(errno));
1209
1210 bp_set_ipopts(s);
1211 bp_bind_ip(s, BFD_DEF_ECHO_PORT);
1212
1213 return s;
1214 }
1215
1216 int bp_echov6_socket(const struct vrf *vrf)
1217 {
1218 int s;
1219
1220 frr_with_privs(&bglobal.bfdd_privs) {
1221 s = vrf_socket(AF_INET6, SOCK_DGRAM, 0, vrf->vrf_id, vrf->name);
1222 }
1223 if (s == -1) {
1224 if (errno != EAFNOSUPPORT)
1225 zlog_fatal("echov6-socket: socket: %s",
1226 strerror(errno));
1227 else
1228 zlog_warn("echov6-socket: V6 is not supported, continuing");
1229
1230 return -1;
1231 }
1232
1233 bp_set_ipv6opts(s);
1234 bp_bind_ipv6(s, BFD_DEF_ECHO_PORT);
1235
1236 return s;
1237 }