2 * Monitor status of frr daemons and restart if necessary.
4 * Copyright (C) 2004 Andrew J. Schorr
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; see the file COPYING; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26 #include <lib/version.h>
29 #include "lib_errors.h"
30 #include "zlog_targets.h"
41 #include "watchfrr_errors.h"
44 #define MIN(X,Y) (((X) <= (Y)) ? (X) : (Y))
47 /* Macros to help randomize timers. */
48 #define JITTER(X) ((frr_weak_random() % ((X)+1))-((X)/2))
49 #define FUZZY(X) ((X)+JITTER((X)/20))
51 #define DEFAULT_PERIOD 5
52 #define DEFAULT_TIMEOUT 90
53 #define DEFAULT_RESTART_TIMEOUT 20
54 #define DEFAULT_LOGLEVEL LOG_INFO
55 #define DEFAULT_MIN_RESTART 60
56 #define DEFAULT_MAX_RESTART 600
58 #define DEFAULT_RESTART_CMD WATCHFRR_SH_PATH " restart %s"
59 #define DEFAULT_START_CMD WATCHFRR_SH_PATH " start %s"
60 #define DEFAULT_STOP_CMD WATCHFRR_SH_PATH " stop %s"
62 #define PING_TOKEN "PING"
64 DEFINE_MGROUP(WATCHFRR
, "watchfrr");
65 DEFINE_MTYPE_STATIC(WATCHFRR
, WATCHFRR_DAEMON
, "watchfrr daemon entry");
67 /* Needs to be global, referenced somewhere inside libfrr. */
68 struct thread_master
*master
;
70 static bool watch_only
= false;
71 const char *pathspace
;
78 PHASE_ZEBRA_RESTART_PENDING
,
79 PHASE_WAITING_ZEBRA_UP
82 static const char *const phase_str
[] = {
86 "Waiting for other daemons to come down",
87 "Zebra restart job running",
88 "Waiting for zebra to come up",
92 #define PHASE_TIMEOUT (3*gs.restart_timeout)
93 #define STARTUP_TIMEOUT 55 * 1000
101 struct thread
*t_kill
;
105 static struct global_state
{
106 enum restart_phase phase
;
107 struct thread
*t_phase_hanging
;
108 struct thread
*t_startup_timeout
;
112 long restart_timeout
;
113 long min_restart_interval
;
114 long max_restart_interval
;
115 struct daemon
*daemons
;
116 const char *restart_command
;
117 const char *start_command
;
118 const char *stop_command
;
119 struct restart_info restart
;
121 struct daemon
*special
; /* points to zebra when doing phased restart */
124 int numdown
; /* # of daemons that are not UP or UNRESPONSIVE */
127 .vtydir
= frr_vtydir
,
128 .period
= 1000 * DEFAULT_PERIOD
,
129 .timeout
= DEFAULT_TIMEOUT
,
130 .restart_timeout
= DEFAULT_RESTART_TIMEOUT
,
131 .loglevel
= DEFAULT_LOGLEVEL
,
132 .min_restart_interval
= DEFAULT_MIN_RESTART
,
133 .max_restart_interval
= DEFAULT_MAX_RESTART
,
134 .restart_command
= DEFAULT_RESTART_CMD
,
135 .start_command
= DEFAULT_START_CMD
,
136 .stop_command
= DEFAULT_STOP_CMD
,
148 (((DMN)->state == DAEMON_UP) || ((DMN)->state == DAEMON_UNRESPONSIVE))
150 static const char *const state_str
[] = {
151 "Init", "Down", "Connecting", "Up", "Unresponsive",
156 enum daemon_state state
;
158 struct timeval echo_sent
;
159 unsigned int connect_tries
;
160 struct thread
*t_wakeup
;
161 struct thread
*t_read
;
162 struct thread
*t_write
;
164 struct restart_info restart
;
167 * For a given daemon, if we've turned on ignore timeouts
168 * ignore the timeout value and assume everything is ok
169 * This is for daemon debugging w/ gdb after we have started
170 * FRR and realize we have something that needs to be looked
176 #define OPTION_MINRESTART 2000
177 #define OPTION_MAXRESTART 2001
178 #define OPTION_DRY 2002
179 #define OPTION_NETNS 2003
181 static const struct option longopts
[] = {
182 {"daemon", no_argument
, NULL
, 'd'},
183 {"statedir", required_argument
, NULL
, 'S'},
184 {"loglevel", required_argument
, NULL
, 'l'},
185 {"interval", required_argument
, NULL
, 'i'},
186 {"timeout", required_argument
, NULL
, 't'},
187 {"restart-timeout", required_argument
, NULL
, 'T'},
188 {"restart", required_argument
, NULL
, 'r'},
189 {"start-command", required_argument
, NULL
, 's'},
190 {"kill-command", required_argument
, NULL
, 'k'},
191 {"dry", no_argument
, NULL
, OPTION_DRY
},
192 {"min-restart-interval", required_argument
, NULL
, OPTION_MINRESTART
},
193 {"max-restart-interval", required_argument
, NULL
, OPTION_MAXRESTART
},
194 {"pid-file", required_argument
, NULL
, 'p'},
195 {"blank-string", required_argument
, NULL
, 'b'},
197 {"netns", optional_argument
, NULL
, OPTION_NETNS
},
199 {"help", no_argument
, NULL
, 'h'},
200 {"version", no_argument
, NULL
, 'v'},
203 static int try_connect(struct daemon
*dmn
);
204 static void wakeup_send_echo(struct thread
*t_wakeup
);
205 static void try_restart(struct daemon
*dmn
);
206 static void phase_check(void);
207 static void restart_done(struct daemon
*dmn
);
209 static const char *progname
;
211 void watchfrr_set_ignore_daemon(struct vty
*vty
, const char *dname
, bool ignore
)
215 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
216 if (strncmp(dmn
->name
, dname
, strlen(dmn
->name
)) == 0)
221 dmn
->ignore_timeout
= ignore
;
222 vty_out(vty
, "%s switching to %s\n", dmn
->name
,
223 ignore
? "ignore" : "watch");
225 vty_out(vty
, "%s is not configured for running at the moment",
229 static void printhelp(FILE *target
)
232 "Usage : %s [OPTION...] <daemon name> ...\n\n\
233 Watchdog program to monitor status of frr daemons and try to restart\n\
234 them if they are down or unresponsive. It determines whether a daemon is\n\
235 up based on whether it can connect to the daemon's vty unix stream socket.\n\
236 It then repeatedly sends echo commands over that socket to determine whether\n\
237 the daemon is responsive. If the daemon crashes, we will receive an EOF\n\
238 on the socket connection and know immediately that the daemon is down.\n\n\
239 The daemons to be monitored should be listed on the command line.\n\n\
240 In order to avoid attempting to restart the daemons in a fast loop,\n\
241 the -m and -M options allow you to control the minimum delay between\n\
242 restart commands. The minimum restart delay is recalculated each time\n\
243 a restart is attempted: if the time since the last restart attempt exceeds\n\
244 twice the -M value, then the restart delay is set to the -m value.\n\
245 Otherwise, the interval is doubled (but capped at the -M value).\n\n",
250 -d, --daemon Run in daemon mode. In this mode, error messages are sent\n\
251 to syslog instead of stdout.\n\
252 -S, --statedir Set the vty socket directory (default is %s)\n\
253 -N, --pathspace Insert prefix into config & socket paths\n"
255 " --netns Create and/or use Linux network namespace. If no name is\n"
256 " given, uses the value from `-N`.\n"
258 "-l, --loglevel Set the logging level (default is %d).\n\
259 The value should range from %d (LOG_EMERG) to %d (LOG_DEBUG),\n\
260 but it can be set higher than %d if extra-verbose debugging\n\
261 messages are desired.\n\
262 --min-restart-interval\n\
263 Set the minimum seconds to wait between invocations of daemon\n\
264 restart commands (default is %d).\n\
265 --max-restart-interval\n\
266 Set the maximum seconds to wait between invocations of daemon\n\
267 restart commands (default is %d).\n\
268 -i, --interval Set the status polling interval in seconds (default is %d)\n\
269 -t, --timeout Set the unresponsiveness timeout in seconds (default is %d)\n\
270 -T, --restart-timeout\n\
271 Set the restart (kill) timeout in seconds (default is %d).\n\
272 If any background jobs are still running after this much\n\
273 time has elapsed, they will be killed.\n\
274 -r, --restart Supply a Bourne shell command to use to restart a single\n\
275 daemon. The command string should include '%%s' where the\n\
276 name of the daemon should be substituted.\n\
278 -s, --start-command\n\
279 Supply a Bourne shell to command to use to start a single\n\
280 daemon. The command string should include '%%s' where the\n\
281 name of the daemon should be substituted.\n\
283 -k, --kill-command\n\
284 Supply a Bourne shell to command to use to stop a single\n\
285 daemon. The command string should include '%%s' where the\n\
286 name of the daemon should be substituted.\n\
288 --dry Do not start or restart anything, just log.\n\
289 -p, --pid-file Set process identifier file name\n\
290 (default is %s/watchfrr.pid).\n\
291 -b, --blank-string\n\
292 When the supplied argument string is found in any of the\n\
293 various shell command arguments (-r, -s, or -k), replace\n\
294 it with a space. This is an ugly hack to circumvent problems\n\
295 passing command-line arguments with embedded spaces.\n\
296 -v, --version Print program version\n\
297 -h, --help Display this help and exit\n",
298 frr_vtydir
, DEFAULT_LOGLEVEL
, LOG_EMERG
, LOG_DEBUG
, LOG_DEBUG
,
299 DEFAULT_MIN_RESTART
, DEFAULT_MAX_RESTART
, DEFAULT_PERIOD
,
300 DEFAULT_TIMEOUT
, DEFAULT_RESTART_TIMEOUT
,
301 DEFAULT_RESTART_CMD
, DEFAULT_START_CMD
, DEFAULT_STOP_CMD
,
305 static pid_t
run_background(char *shell_cmd
)
309 switch (child
= fork()) {
311 flog_err_sys(EC_LIB_SYSTEM_CALL
,
312 "fork failed, cannot run command [%s]: %s",
313 shell_cmd
, safe_strerror(errno
));
317 /* Use separate process group so child processes can be killed
319 if (setpgid(0, 0) < 0)
320 zlog_warn("setpgid(0,0) failed: %s",
321 safe_strerror(errno
));
325 char *const argv
[4] = {shell
, dashc
, shell_cmd
, NULL
};
326 execv("/bin/sh", argv
);
327 flog_err_sys(EC_LIB_SYSTEM_CALL
,
328 "execv(/bin/sh -c '%s') failed: %s",
329 shell_cmd
, safe_strerror(errno
));
333 /* Parent process: we will reap the child later. */
334 zlog_info("Forked background command [pid %d]: %s", (int)child
,
340 static struct timeval
*time_elapsed(struct timeval
*result
,
341 const struct timeval
*start_time
)
343 gettimeofday(result
, NULL
);
344 result
->tv_sec
-= start_time
->tv_sec
;
345 result
->tv_usec
-= start_time
->tv_usec
;
346 while (result
->tv_usec
< 0) {
347 result
->tv_usec
+= 1000000L;
353 static void restart_kill(struct thread
*t_kill
)
355 struct restart_info
*restart
= THREAD_ARG(t_kill
);
356 struct timeval delay
;
358 time_elapsed(&delay
, &restart
->time
);
360 "%s %s child process %d still running after %ld seconds, sending signal %d",
361 restart
->what
, restart
->name
, (int)restart
->pid
,
362 (long)delay
.tv_sec
, (restart
->kills
? SIGKILL
: SIGTERM
));
363 kill(-restart
->pid
, (restart
->kills
? SIGKILL
: SIGTERM
));
365 thread_add_timer(master
, restart_kill
, restart
, gs
.restart_timeout
,
369 static struct restart_info
*find_child(pid_t child
)
372 if (gs
.restart
.pid
== child
)
375 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
376 if (dmn
->restart
.pid
== child
)
377 return &dmn
->restart
;
382 static void sigchild(void)
388 struct restart_info
*restart
;
391 switch (child
= waitpid(-1, &status
, WNOHANG
)) {
393 flog_err_sys(EC_LIB_SYSTEM_CALL
, "waitpid failed: %s",
394 safe_strerror(errno
));
397 zlog_warn("SIGCHLD received, but waitpid did not reap a child");
401 if (child
== integrated_write_pid
) {
402 integrated_write_sigchld(status
);
406 if ((restart
= find_child(child
)) != NULL
) {
407 name
= restart
->name
;
408 what
= restart
->what
;
411 thread_cancel(&restart
->t_kill
);
413 /* Update restart time to reflect the time the command
415 gettimeofday(&restart
->time
, NULL
);
419 "waitpid returned status for an unknown child process %d",
424 if (WIFSTOPPED(status
))
425 zlog_warn("%s %s process %d is stopped", what
, name
,
427 else if (WIFSIGNALED(status
))
428 zlog_warn("%s %s process %d terminated due to signal %d", what
,
429 name
, (int)child
, WTERMSIG(status
));
430 else if (WIFEXITED(status
)) {
431 if (WEXITSTATUS(status
) != 0)
433 "%s %s process %d exited with non-zero status %d",
434 what
, name
, (int)child
, WEXITSTATUS(status
));
436 zlog_debug("%s %s process %d exited normally", what
,
439 if (restart
&& restart
!= &gs
.restart
) {
440 dmn
= container_of(restart
, struct daemon
,
444 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
450 "cannot interpret %s %s process %d wait status 0x%x",
451 what
, name
, (int)child
, status
);
455 static int run_job(struct restart_info
*restart
, const char *cmdtype
,
456 const char *command
, int force
, int update_interval
)
458 struct timeval delay
;
460 if (gs
.loglevel
> LOG_DEBUG
+ 1)
461 zlog_debug("attempting to %s %s", cmdtype
, restart
->name
);
464 if (gs
.loglevel
> LOG_DEBUG
+ 1)
466 "cannot %s %s, previous pid %d still running",
467 cmdtype
, restart
->name
, (int)restart
->pid
);
473 snprintf(buffer
, sizeof(buffer
), "restarting %s", restart
->name
);
474 systemd_send_status(buffer
);
476 /* Note: time_elapsed test must come before the force test, since we
478 to make sure that delay is initialized for use below in updating the
480 if ((time_elapsed(&delay
, &restart
->time
)->tv_sec
< restart
->interval
)
483 if (gs
.loglevel
> LOG_DEBUG
+ 1)
485 "postponing %s %s: elapsed time %ld < retry interval %ld",
486 cmdtype
, restart
->name
, (long)delay
.tv_sec
,
491 gettimeofday(&restart
->time
, NULL
);
494 char cmd
[strlen(command
) + strlen(restart
->name
) + 1];
495 snprintf(cmd
, sizeof(cmd
), command
, restart
->name
);
496 if ((restart
->pid
= run_background(cmd
)) > 0) {
497 thread_add_timer(master
, restart_kill
, restart
,
498 gs
.restart_timeout
, &restart
->t_kill
);
499 restart
->what
= cmdtype
;
505 systemd_send_status("FRR Operational");
507 /* Calculate the new restart interval. */
508 if (update_interval
) {
509 if (delay
.tv_sec
> 2 * gs
.max_restart_interval
)
510 restart
->interval
= gs
.min_restart_interval
;
511 else if ((restart
->interval
*= 2) > gs
.max_restart_interval
)
512 restart
->interval
= gs
.max_restart_interval
;
513 if (gs
.loglevel
> LOG_DEBUG
+ 1)
514 zlog_debug("restart %s interval is now %ld",
515 restart
->name
, restart
->interval
);
520 #define SET_READ_HANDLER(DMN) \
522 (DMN)->t_read = NULL; \
523 thread_add_read(master, handle_read, (DMN), (DMN)->fd, \
527 #define SET_WAKEUP_DOWN(DMN) \
529 (DMN)->t_wakeup = NULL; \
530 thread_add_timer_msec(master, wakeup_down, (DMN), \
531 FUZZY(gs.period), &(DMN)->t_wakeup); \
534 #define SET_WAKEUP_UNRESPONSIVE(DMN) \
536 (DMN)->t_wakeup = NULL; \
537 thread_add_timer_msec(master, wakeup_unresponsive, (DMN), \
538 FUZZY(gs.period), &(DMN)->t_wakeup); \
541 #define SET_WAKEUP_ECHO(DMN) \
543 (DMN)->t_wakeup = NULL; \
544 thread_add_timer_msec(master, wakeup_send_echo, (DMN), \
545 FUZZY(gs.period), &(DMN)->t_wakeup); \
548 static void wakeup_down(struct thread
*t_wakeup
)
550 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
552 dmn
->t_wakeup
= NULL
;
553 if (try_connect(dmn
) < 0)
554 SET_WAKEUP_DOWN(dmn
);
555 if ((dmn
->connect_tries
> 1) && (dmn
->state
!= DAEMON_UP
))
559 static void wakeup_init(struct thread
*t_wakeup
)
561 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
563 dmn
->t_wakeup
= NULL
;
564 if (try_connect(dmn
) < 0) {
566 "%s state -> down : initial connection attempt failed",
568 dmn
->state
= DAEMON_DOWN
;
573 static void restart_done(struct daemon
*dmn
)
575 if (dmn
->state
!= DAEMON_DOWN
) {
577 "Daemon: %s: is in %s state but expected it to be in DAEMON_DOWN state",
578 dmn
->name
, state_str
[dmn
->state
]);
581 THREAD_OFF(dmn
->t_wakeup
);
583 if (try_connect(dmn
) < 0)
584 SET_WAKEUP_DOWN(dmn
);
587 static void daemon_down(struct daemon
*dmn
, const char *why
)
589 if (IS_UP(dmn
) || (dmn
->state
== DAEMON_INIT
))
590 flog_err(EC_WATCHFRR_CONNECTION
, "%s state -> down : %s",
592 else if (gs
.loglevel
> LOG_DEBUG
)
593 zlog_debug("%s still down : %s", dmn
->name
, why
);
596 dmn
->state
= DAEMON_DOWN
;
601 THREAD_OFF(dmn
->t_read
);
602 THREAD_OFF(dmn
->t_write
);
603 THREAD_OFF(dmn
->t_wakeup
);
604 if (try_connect(dmn
) < 0)
605 SET_WAKEUP_DOWN(dmn
);
609 static void handle_read(struct thread
*t_read
)
611 struct daemon
*dmn
= THREAD_ARG(t_read
);
612 static const char resp
[sizeof(PING_TOKEN
) + 4] = PING_TOKEN
"\n";
613 char buf
[sizeof(resp
) + 100];
615 struct timeval delay
;
618 if ((rc
= read(dmn
->fd
, buf
, sizeof(buf
))) < 0) {
621 if (ERRNO_IO_RETRY(errno
)) {
622 /* Pretend it never happened. */
623 SET_READ_HANDLER(dmn
);
626 snprintf(why
, sizeof(why
), "unexpected read error: %s",
627 safe_strerror(errno
));
628 daemon_down(dmn
, why
);
632 daemon_down(dmn
, "read returned EOF");
635 if (!dmn
->echo_sent
.tv_sec
) {
636 char why
[sizeof(buf
) + 100];
637 snprintf(why
, sizeof(why
),
638 "unexpected read returns %d bytes: %.*s", (int)rc
,
640 daemon_down(dmn
, why
);
644 /* We are expecting an echo response: is there any chance that the
645 response would not be returned entirely in the first read? That
646 seems inconceivable... */
647 if ((rc
!= sizeof(resp
)) || memcmp(buf
, resp
, sizeof(resp
))) {
648 char why
[100 + sizeof(buf
)];
649 snprintf(why
, sizeof(why
),
650 "read returned bad echo response of %d bytes (expecting %u): %.*s",
651 (int)rc
, (unsigned int)sizeof(resp
), (int)rc
, buf
);
652 daemon_down(dmn
, why
);
656 time_elapsed(&delay
, &dmn
->echo_sent
);
657 dmn
->echo_sent
.tv_sec
= 0;
658 if (dmn
->state
== DAEMON_UNRESPONSIVE
) {
659 if (delay
.tv_sec
< gs
.timeout
) {
660 dmn
->state
= DAEMON_UP
;
662 "%s state -> up : echo response received after %ld.%06ld seconds",
663 dmn
->name
, (long)delay
.tv_sec
,
664 (long)delay
.tv_usec
);
667 "%s: slow echo response finally received after %ld.%06ld seconds",
668 dmn
->name
, (long)delay
.tv_sec
,
669 (long)delay
.tv_usec
);
670 } else if (gs
.loglevel
> LOG_DEBUG
+ 1)
671 zlog_debug("%s: echo response received after %ld.%06ld seconds",
672 dmn
->name
, (long)delay
.tv_sec
, (long)delay
.tv_usec
);
674 SET_READ_HANDLER(dmn
);
675 thread_cancel(&dmn
->t_wakeup
);
676 SET_WAKEUP_ECHO(dmn
);
680 * Wait till we notice that all daemons are ready before
681 * we send we are ready to systemd
683 static void daemon_send_ready(int exitcode
)
693 zlog_notice("all daemons up, doing startup-complete notify");
694 else if (gs
.numdown
< gs
.numdaemons
)
695 flog_err(EC_WATCHFRR_CONNECTION
,
696 "startup did not complete within timeout (%d/%d daemons running)",
697 gs
.numdaemons
- gs
.numdown
, gs
.numdaemons
);
699 flog_err(EC_WATCHFRR_CONNECTION
,
700 "all configured daemons failed to start -- exiting watchfrr");
707 snprintf(started
, sizeof(started
), "%s/%s", frr_vtydir
,
709 fp
= fopen(started
, "w");
713 systemd_send_started(master
);
714 systemd_send_status("FRR Operational");
718 static void daemon_up(struct daemon
*dmn
, const char *why
)
720 dmn
->state
= DAEMON_UP
;
722 dmn
->connect_tries
= 0;
723 zlog_notice("%s state -> up : %s", dmn
->name
, why
);
725 daemon_send_ready(0);
726 SET_WAKEUP_ECHO(dmn
);
730 static void check_connect(struct thread
*t_write
)
732 struct daemon
*dmn
= THREAD_ARG(t_write
);
734 socklen_t reslen
= sizeof(sockerr
);
737 if (getsockopt(dmn
->fd
, SOL_SOCKET
, SO_ERROR
, (char *)&sockerr
, &reslen
)
739 zlog_warn("%s: check_connect: getsockopt failed: %s", dmn
->name
,
740 safe_strerror(errno
));
742 "getsockopt failed checking connection success");
745 if ((reslen
== sizeof(sockerr
)) && sockerr
) {
749 "getsockopt reports that connection attempt failed: %s",
750 safe_strerror(sockerr
));
751 daemon_down(dmn
, why
);
755 daemon_up(dmn
, "delayed connect succeeded");
758 static void wakeup_connect_hanging(struct thread
*t_wakeup
)
760 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
763 dmn
->t_wakeup
= NULL
;
764 snprintf(why
, sizeof(why
),
765 "connection attempt timed out after %ld seconds", gs
.timeout
);
766 daemon_down(dmn
, why
);
769 /* Making connection to protocol daemon. */
770 static int try_connect(struct daemon
*dmn
)
773 struct sockaddr_un addr
;
776 if (gs
.loglevel
> LOG_DEBUG
+ 1)
777 zlog_debug("%s: attempting to connect", dmn
->name
);
778 dmn
->connect_tries
++;
780 memset(&addr
, 0, sizeof(struct sockaddr_un
));
781 addr
.sun_family
= AF_UNIX
;
782 snprintf(addr
.sun_path
, sizeof(addr
.sun_path
), "%s/%s.vty", gs
.vtydir
,
784 #ifdef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN
785 len
= addr
.sun_len
= SUN_LEN(&addr
);
787 len
= sizeof(addr
.sun_family
) + strlen(addr
.sun_path
);
788 #endif /* HAVE_STRUCT_SOCKADDR_UN_SUN_LEN */
790 /* Quick check to see if we might succeed before we go to the trouble
791 of creating a socket. */
792 if (access(addr
.sun_path
, W_OK
) < 0) {
794 flog_err_sys(EC_LIB_SYSTEM_CALL
,
795 "%s: access to socket %s denied: %s",
796 dmn
->name
, addr
.sun_path
,
797 safe_strerror(errno
));
801 if ((sock
= socket(AF_UNIX
, SOCK_STREAM
, 0)) < 0) {
802 flog_err_sys(EC_LIB_SOCKET
, "%s(%s): cannot make socket: %s",
803 __func__
, addr
.sun_path
, safe_strerror(errno
));
807 if (set_nonblocking(sock
) < 0 || set_cloexec(sock
) < 0) {
808 flog_err_sys(EC_LIB_SYSTEM_CALL
,
809 "%s(%s): set_nonblocking/cloexec(%d) failed",
810 __func__
, addr
.sun_path
, sock
);
815 if (connect(sock
, (struct sockaddr
*)&addr
, len
) < 0) {
816 if ((errno
!= EINPROGRESS
) && (errno
!= EWOULDBLOCK
)) {
817 if (gs
.loglevel
> LOG_DEBUG
)
818 zlog_debug("%s(%s): connect failed: %s",
819 __func__
, addr
.sun_path
,
820 safe_strerror(errno
));
824 if (gs
.loglevel
> LOG_DEBUG
)
825 zlog_debug("%s: connection in progress", dmn
->name
);
826 dmn
->state
= DAEMON_CONNECTING
;
828 thread_add_write(master
, check_connect
, dmn
, dmn
->fd
,
830 thread_add_timer(master
, wakeup_connect_hanging
, dmn
,
831 gs
.timeout
, &dmn
->t_wakeup
);
832 SET_READ_HANDLER(dmn
);
837 SET_READ_HANDLER(dmn
);
838 daemon_up(dmn
, "connect succeeded");
842 static void phase_hanging(struct thread
*t_hanging
)
844 gs
.t_phase_hanging
= NULL
;
845 flog_err(EC_WATCHFRR_CONNECTION
,
846 "Phase [%s] hanging for %ld seconds, aborting phased restart",
847 phase_str
[gs
.phase
], PHASE_TIMEOUT
);
848 gs
.phase
= PHASE_NONE
;
851 static void set_phase(enum restart_phase new_phase
)
853 gs
.phase
= new_phase
;
854 thread_cancel(&gs
.t_phase_hanging
);
856 thread_add_timer(master
, phase_hanging
, NULL
, PHASE_TIMEOUT
,
857 &gs
.t_phase_hanging
);
860 static void phase_check(void)
869 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
870 if (dmn
->state
== DAEMON_INIT
)
873 /* startup complete, everything out of INIT */
874 gs
.phase
= PHASE_NONE
;
875 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
876 if (dmn
->state
== DAEMON_DOWN
) {
877 SET_WAKEUP_DOWN(dmn
);
881 case PHASE_STOPS_PENDING
:
885 "Phased restart: all routing daemon stop jobs have completed.");
886 set_phase(PHASE_WAITING_DOWN
);
889 case PHASE_WAITING_DOWN
:
890 if (gs
.numdown
+ IS_UP(gs
.special
) < gs
.numdaemons
)
892 zlog_info("Phased restart: all routing daemons now down.");
893 run_job(&gs
.special
->restart
, "restart", gs
.restart_command
, 1,
895 set_phase(PHASE_ZEBRA_RESTART_PENDING
);
898 case PHASE_ZEBRA_RESTART_PENDING
:
899 if (gs
.special
->restart
.pid
)
901 zlog_info("Phased restart: %s restart job completed.",
903 set_phase(PHASE_WAITING_ZEBRA_UP
);
906 case PHASE_WAITING_ZEBRA_UP
:
907 if (!IS_UP(gs
.special
))
909 zlog_info("Phased restart: %s is now up.", gs
.special
->name
);
912 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
913 if (dmn
!= gs
.special
)
914 run_job(&dmn
->restart
, "start",
915 gs
.start_command
, 1, 0);
918 gs
.phase
= PHASE_NONE
;
919 THREAD_OFF(gs
.t_phase_hanging
);
920 zlog_notice("Phased global restart has completed.");
925 static void try_restart(struct daemon
*dmn
)
930 if (dmn
!= gs
.special
) {
931 if ((gs
.special
->state
== DAEMON_UP
)
932 && (gs
.phase
== PHASE_NONE
))
933 run_job(&dmn
->restart
, "restart", gs
.restart_command
, 0,
937 "%s: postponing restart attempt because master %s daemon not up [%s], or phased restart in progress",
938 dmn
->name
, gs
.special
->name
,
939 state_str
[gs
.special
->state
]);
943 if ((gs
.phase
!= PHASE_NONE
) || gs
.numpids
) {
944 if (gs
.loglevel
> LOG_DEBUG
+ 1)
946 "postponing phased global restart: restart already in progress [%s], or outstanding child processes [%d]",
947 phase_str
[gs
.phase
], gs
.numpids
);
950 /* Is it too soon for a restart? */
952 struct timeval delay
;
953 if (time_elapsed(&delay
, &gs
.special
->restart
.time
)->tv_sec
954 < gs
.special
->restart
.interval
) {
955 if (gs
.loglevel
> LOG_DEBUG
+ 1)
957 "postponing phased global restart: elapsed time %ld < retry interval %ld",
959 gs
.special
->restart
.interval
);
963 run_job(&gs
.restart
, "restart", gs
.restart_command
, 0, 1);
966 static void wakeup_unresponsive(struct thread
*t_wakeup
)
968 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
970 dmn
->t_wakeup
= NULL
;
971 if (dmn
->state
!= DAEMON_UNRESPONSIVE
)
972 flog_err(EC_WATCHFRR_CONNECTION
,
973 "%s: no longer unresponsive (now %s), wakeup should have been cancelled!",
974 dmn
->name
, state_str
[dmn
->state
]);
976 SET_WAKEUP_UNRESPONSIVE(dmn
);
981 static void wakeup_no_answer(struct thread
*t_wakeup
)
983 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
985 dmn
->t_wakeup
= NULL
;
986 dmn
->state
= DAEMON_UNRESPONSIVE
;
987 if (dmn
->ignore_timeout
)
989 flog_err(EC_WATCHFRR_CONNECTION
,
990 "%s state -> unresponsive : no response yet to ping sent %ld seconds ago",
991 dmn
->name
, gs
.timeout
);
992 SET_WAKEUP_UNRESPONSIVE(dmn
);
996 static void wakeup_send_echo(struct thread
*t_wakeup
)
998 static const char echocmd
[] = "echo " PING_TOKEN
;
1000 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
1002 dmn
->t_wakeup
= NULL
;
1003 if (((rc
= write(dmn
->fd
, echocmd
, sizeof(echocmd
))) < 0)
1004 || ((size_t)rc
!= sizeof(echocmd
))) {
1005 char why
[100 + sizeof(echocmd
)];
1006 snprintf(why
, sizeof(why
),
1007 "write '%s' returned %d instead of %u", echocmd
,
1008 (int)rc
, (unsigned int)sizeof(echocmd
));
1009 daemon_down(dmn
, why
);
1011 gettimeofday(&dmn
->echo_sent
, NULL
);
1012 thread_add_timer(master
, wakeup_no_answer
, dmn
, gs
.timeout
,
1017 bool check_all_up(void)
1021 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
1022 if (dmn
->state
!= DAEMON_UP
)
1027 void watchfrr_status(struct vty
*vty
)
1030 struct timeval delay
;
1032 vty_out(vty
, "watchfrr global phase: %s\n", phase_str
[gs
.phase
]);
1034 vty_out(vty
, " global restart running, pid %ld\n",
1035 (long)gs
.restart
.pid
);
1037 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
1038 vty_out(vty
, " %-20s %s%s", dmn
->name
, state_str
[dmn
->state
],
1039 dmn
->ignore_timeout
? "/Ignoring Timeout\n" : "\n");
1040 if (dmn
->restart
.pid
)
1041 vty_out(vty
, " restart running, pid %ld\n",
1042 (long)dmn
->restart
.pid
);
1043 else if (dmn
->state
== DAEMON_DOWN
&&
1044 time_elapsed(&delay
, &dmn
->restart
.time
)->tv_sec
1045 < dmn
->restart
.interval
)
1046 vty_out(vty
, " restarting in %jd seconds (%jds backoff interval)\n",
1047 (intmax_t)dmn
->restart
.interval
1048 - (intmax_t)delay
.tv_sec
,
1049 (intmax_t)dmn
->restart
.interval
);
1053 static void sigint(void)
1055 zlog_notice("Terminating on signal");
1056 systemd_send_stopping();
1060 static int valid_command(const char *cmd
)
1067 return ((p
= strchr(cmd
, '%')) != NULL
) && (*(p
+ 1) == 's')
1068 && !strchr(p
+ 1, '%');
1071 /* This is an ugly hack to circumvent problems with passing command-line
1072 arguments that contain spaces. The fix is to use a configuration file. */
1073 static char *translate_blanks(const char *cmd
, const char *blankstr
)
1077 size_t bslen
= strlen(blankstr
);
1079 if (!(res
= strdup(cmd
))) {
1083 while ((p
= strstr(res
, blankstr
)) != NULL
) {
1086 memmove(p
+ 1, p
+ bslen
, strlen(p
+ bslen
) + 1);
1091 static void startup_timeout(struct thread
*t_wakeup
)
1093 daemon_send_ready(1);
1098 #include <sys/mount.h>
1101 #define NETNS_RUN_DIR "/var/run/netns"
1103 static void netns_create(int dirfd
, const char *nsname
)
1105 /* make /var/run/netns shared between mount namespaces
1106 * just like iproute2 sets it up
1108 if (mount("", NETNS_RUN_DIR
, "none", MS_SHARED
| MS_REC
, NULL
)) {
1109 if (errno
!= EINVAL
) {
1114 if (mount(NETNS_RUN_DIR
, NETNS_RUN_DIR
, "none",
1115 MS_BIND
| MS_REC
, NULL
)) {
1120 if (mount("", NETNS_RUN_DIR
, "none", MS_SHARED
| MS_REC
,
1127 /* need an empty file to mount on top of */
1128 int nsfd
= openat(dirfd
, nsname
, O_CREAT
| O_RDONLY
| O_EXCL
, 0);
1131 fprintf(stderr
, "failed to create \"%s/%s\": %s\n",
1132 NETNS_RUN_DIR
, nsname
, strerror(errno
));
1137 if (unshare(CLONE_NEWNET
)) {
1139 unlinkat(dirfd
, nsname
, 0);
1143 char *dstpath
= asprintfrr(MTYPE_TMP
, "%s/%s", NETNS_RUN_DIR
, nsname
);
1145 /* bind-mount so the namespace has a name and is persistent */
1146 if (mount("/proc/self/ns/net", dstpath
, "none", MS_BIND
, NULL
) < 0) {
1147 fprintf(stderr
, "failed to bind-mount netns to \"%s\": %s\n",
1148 dstpath
, strerror(errno
));
1149 unlinkat(dirfd
, nsname
, 0);
1153 XFREE(MTYPE_TMP
, dstpath
);
1156 static void netns_setup(const char *nsname
)
1160 dirfd
= open(NETNS_RUN_DIR
, O_DIRECTORY
| O_RDONLY
);
1162 if (errno
== ENOTDIR
) {
1163 fprintf(stderr
, "error: \"%s\" is not a directory!\n",
1166 } else if (errno
== ENOENT
) {
1167 if (mkdir(NETNS_RUN_DIR
, 0755)) {
1168 fprintf(stderr
, "error: \"%s\": mkdir: %s\n",
1169 NETNS_RUN_DIR
, strerror(errno
));
1172 dirfd
= open(NETNS_RUN_DIR
, O_DIRECTORY
| O_RDONLY
);
1174 fprintf(stderr
, "error: \"%s\": opendir: %s\n",
1175 NETNS_RUN_DIR
, strerror(errno
));
1179 fprintf(stderr
, "error: \"%s\": %s\n",
1180 NETNS_RUN_DIR
, strerror(errno
));
1185 nsfd
= openat(dirfd
, nsname
, O_RDONLY
);
1186 if (nsfd
< 0 && errno
!= ENOENT
) {
1187 fprintf(stderr
, "error: \"%s/%s\": %s\n",
1188 NETNS_RUN_DIR
, nsname
, strerror(errno
));
1192 netns_create(dirfd
, nsname
);
1194 if (setns(nsfd
, CLONE_NEWNET
)) {
1202 /* make sure loopback is up... weird things happen otherwise.
1203 * ioctl is perfectly fine for this, don't need netlink...
1206 struct ifreq ifr
= { };
1208 strlcpy(ifr
.ifr_name
, "lo", sizeof(ifr
.ifr_name
));
1210 sockfd
= socket(AF_INET
, SOCK_DGRAM
, 0);
1215 if (ioctl(sockfd
, SIOCGIFFLAGS
, &ifr
)) {
1216 perror("ioctl(SIOCGIFFLAGS, \"lo\")");
1219 if (!(ifr
.ifr_flags
& IFF_UP
)) {
1220 ifr
.ifr_flags
|= IFF_UP
;
1221 if (ioctl(sockfd
, SIOCSIFFLAGS
, &ifr
)) {
1222 perror("ioctl(SIOCSIFFLAGS, \"lo\")");
1229 #else /* !GNU_LINUX */
1231 static void netns_setup(const char *nsname
)
1233 fprintf(stderr
, "network namespaces are only available on Linux\n");
1238 static void watchfrr_init(int argc
, char **argv
)
1240 const char *special
= "zebra";
1242 struct daemon
*dmn
, **add
= &gs
.daemons
;
1243 char alldaemons
[512] = "", *p
= alldaemons
;
1245 thread_add_timer_msec(master
, startup_timeout
, NULL
, STARTUP_TIMEOUT
,
1246 &gs
.t_startup_timeout
);
1248 for (i
= optind
; i
< argc
; i
++) {
1249 dmn
= XCALLOC(MTYPE_WATCHFRR_DAEMON
, sizeof(*dmn
));
1251 dmn
->name
= dmn
->restart
.name
= argv
[i
];
1252 dmn
->state
= DAEMON_INIT
;
1256 thread_add_timer_msec(master
, wakeup_init
, dmn
, 0,
1258 dmn
->restart
.interval
= gs
.min_restart_interval
;
1262 if (!strcmp(dmn
->name
, special
))
1268 "Must specify one or more daemons to monitor.\n\n");
1271 if (!watch_only
&& !gs
.special
) {
1272 fprintf(stderr
, "\"%s\" daemon must be in daemon lists\n\n",
1277 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
1278 snprintf(p
, alldaemons
+ sizeof(alldaemons
) - p
, "%s%s",
1279 (p
== alldaemons
) ? "" : " ", dmn
->name
);
1282 zlog_notice("%s %s watching [%s]%s", progname
, FRR_VERSION
, alldaemons
,
1283 watch_only
? ", monitor mode" : "");
1286 struct zebra_privs_t watchfrr_privs
= {
1288 .vty_group
= VTY_GROUP
,
1292 static struct frr_signal_t watchfrr_signals
[] = {
1303 .handler
= sigchild
,
1307 FRR_DAEMON_INFO(watchfrr
, WATCHFRR
,
1308 .flags
= FRR_NO_PRIVSEP
| FRR_NO_TCPVTY
| FRR_LIMITED_CLI
1309 | FRR_NO_CFG_PID_DRY
| FRR_NO_ZCLIENT
1312 .printhelp
= printhelp
,
1313 .copyright
= "Copyright 2004 Andrew J. Schorr",
1315 .signals
= watchfrr_signals
,
1316 .n_signals
= array_size(watchfrr_signals
),
1318 .privs
= &watchfrr_privs
,
1321 #define DEPRECATED_OPTIONS "aAezR:"
1323 int main(int argc
, char **argv
)
1326 const char *blankstr
= NULL
;
1327 const char *netns
= NULL
;
1328 bool netns_en
= false;
1330 frr_preinit(&watchfrr_di
, argc
, argv
);
1331 progname
= watchfrr_di
.progname
;
1333 frr_opt_add("b:di:k:l:N:p:r:S:s:t:T:" DEPRECATED_OPTIONS
, longopts
, "");
1335 gs
.restart
.name
= "all";
1336 while ((opt
= frr_getopt(argc
, argv
, NULL
)) != EOF
) {
1337 if (opt
&& opt
< 128 && strchr(DEPRECATED_OPTIONS
, opt
)) {
1339 "The -%c option no longer exists.\n"
1340 "Please refer to the watchfrr(8) man page.\n",
1355 if (!valid_command(optarg
)) {
1357 "Invalid kill command, must contain '%%s': %s\n",
1361 gs
.stop_command
= optarg
;
1365 if ((sscanf(optarg
, "%d%1s", &gs
.loglevel
, garbage
)
1367 || (gs
.loglevel
< LOG_EMERG
)) {
1369 "Invalid loglevel argument: %s\n",
1374 case OPTION_MINRESTART
: {
1376 if ((sscanf(optarg
, "%ld%1s", &gs
.min_restart_interval
,
1379 || (gs
.min_restart_interval
< 0)) {
1381 "Invalid min_restart_interval argument: %s\n",
1386 case OPTION_MAXRESTART
: {
1388 if ((sscanf(optarg
, "%ld%1s", &gs
.max_restart_interval
,
1391 || (gs
.max_restart_interval
< 0)) {
1393 "Invalid max_restart_interval argument: %s\n",
1400 if (optarg
&& strchr(optarg
, '/')) {
1402 "invalid network namespace name \"%s\" (may not contain slashes)\n",
1411 if ((sscanf(optarg
, "%d%1s", &period
, garbage
) != 1)
1412 || (gs
.period
< 1)) {
1414 "Invalid interval argument: %s\n",
1418 gs
.period
= 1000 * period
;
1421 watchfrr_di
.pid_file
= optarg
;
1424 if (!valid_command(optarg
)) {
1426 "Invalid restart command, must contain '%%s': %s\n",
1430 gs
.restart_command
= optarg
;
1433 if (!valid_command(optarg
)) {
1435 "Invalid start command, must contain '%%s': %s\n",
1439 gs
.start_command
= optarg
;
1446 if ((sscanf(optarg
, "%ld%1s", &gs
.timeout
, garbage
)
1448 || (gs
.timeout
< 1)) {
1450 "Invalid timeout argument: %s\n",
1457 if ((sscanf(optarg
, "%ld%1s", &gs
.restart_timeout
,
1460 || (gs
.restart_timeout
< 1)) {
1462 "Invalid restart timeout argument: %s\n",
1468 fputs("Invalid option.\n", stderr
);
1474 && (gs
.start_command
|| gs
.stop_command
|| gs
.restart_command
)) {
1475 fputs("Options -r/-s/-k are not used when --dry is active.\n",
1479 && (!gs
.restart_command
|| !gs
.start_command
|| !gs
.stop_command
)) {
1481 "Options -s (start), -k (kill), and -r (restart) are required.\n");
1486 if (gs
.restart_command
)
1487 gs
.restart_command
=
1488 translate_blanks(gs
.restart_command
, blankstr
);
1489 if (gs
.start_command
)
1491 translate_blanks(gs
.start_command
, blankstr
);
1492 if (gs
.stop_command
)
1494 translate_blanks(gs
.stop_command
, blankstr
);
1497 gs
.restart
.interval
= gs
.min_restart_interval
;
1499 /* env variable for the processes that we start */
1500 if (watchfrr_di
.pathspace
)
1501 setenv("FRR_PATHSPACE", watchfrr_di
.pathspace
, 1);
1503 unsetenv("FRR_PATHSPACE");
1506 * when watchfrr_di.pathspace is read, if it is not specified
1507 * pathspace is NULL as expected
1509 pathspace
= watchfrr_di
.pathspace
;
1511 if (netns_en
&& !netns
)
1512 netns
= watchfrr_di
.pathspace
;
1514 if (netns_en
&& netns
&& netns
[0])
1517 master
= frr_init();
1518 watchfrr_error_init();
1519 watchfrr_init(argc
, argv
);
1520 watchfrr_vty_init();
1524 if (watchfrr_di
.daemon_mode
)
1525 zlog_syslog_set_prio_min(MIN(gs
.loglevel
, LOG_DEBUG
));
1527 zlog_aux_init(NULL
, MIN(gs
.loglevel
, LOG_DEBUG
));
1531 systemd_send_stopping();