2 * Monitor status of frr daemons and restart if necessary.
4 * Copyright (C) 2004 Andrew J. Schorr
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; see the file COPYING; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26 #include <lib/version.h>
29 #include "lib_errors.h"
38 #include "watchfrr_errors.h"
41 #define MIN(X,Y) (((X) <= (Y)) ? (X) : (Y))
44 /* Macros to help randomize timers. */
45 #define JITTER(X) ((random() % ((X)+1))-((X)/2))
46 #define FUZZY(X) ((X)+JITTER((X)/20))
48 #define DEFAULT_PERIOD 5
49 #define DEFAULT_TIMEOUT 90
50 #define DEFAULT_RESTART_TIMEOUT 20
51 #define DEFAULT_LOGLEVEL LOG_INFO
52 #define DEFAULT_MIN_RESTART 60
53 #define DEFAULT_MAX_RESTART 600
55 #define DEFAULT_RESTART_CMD WATCHFRR_SH_PATH " restart %s"
56 #define DEFAULT_START_CMD WATCHFRR_SH_PATH " start %s"
57 #define DEFAULT_STOP_CMD WATCHFRR_SH_PATH " stop %s"
59 #define PING_TOKEN "PING"
61 DEFINE_MGROUP(WATCHFRR
, "watchfrr")
62 DEFINE_MTYPE_STATIC(WATCHFRR
, WATCHFRR_DAEMON
, "watchfrr daemon entry")
64 /* Needs to be global, referenced somewhere inside libfrr. */
65 struct thread_master
*master
;
67 static bool watch_only
= false;
74 PHASE_ZEBRA_RESTART_PENDING
,
75 PHASE_WAITING_ZEBRA_UP
78 static const char *const phase_str
[] = {
82 "Waiting for other daemons to come down",
83 "Zebra restart job running",
84 "Waiting for zebra to come up",
88 #define PHASE_TIMEOUT (3*gs.restart_timeout)
89 #define STARTUP_TIMEOUT 55 * 1000
97 struct thread
*t_kill
;
101 static struct global_state
{
102 restart_phase_t phase
;
103 struct thread
*t_phase_hanging
;
104 struct thread
*t_startup_timeout
;
108 long restart_timeout
;
109 long min_restart_interval
;
110 long max_restart_interval
;
111 struct daemon
*daemons
;
112 const char *restart_command
;
113 const char *start_command
;
114 const char *stop_command
;
115 struct restart_info restart
;
117 struct daemon
*special
; /* points to zebra when doing phased restart */
120 int numdown
; /* # of daemons that are not UP or UNRESPONSIVE */
123 .vtydir
= frr_vtydir
,
124 .period
= 1000 * DEFAULT_PERIOD
,
125 .timeout
= DEFAULT_TIMEOUT
,
126 .restart_timeout
= DEFAULT_RESTART_TIMEOUT
,
127 .loglevel
= DEFAULT_LOGLEVEL
,
128 .min_restart_interval
= DEFAULT_MIN_RESTART
,
129 .max_restart_interval
= DEFAULT_MAX_RESTART
,
130 .restart_command
= DEFAULT_RESTART_CMD
,
131 .start_command
= DEFAULT_START_CMD
,
132 .stop_command
= DEFAULT_STOP_CMD
,
144 (((DMN)->state == DAEMON_UP) || ((DMN)->state == DAEMON_UNRESPONSIVE))
146 static const char *const state_str
[] = {
147 "Init", "Down", "Connecting", "Up", "Unresponsive",
152 daemon_state_t state
;
154 struct timeval echo_sent
;
155 unsigned int connect_tries
;
156 struct thread
*t_wakeup
;
157 struct thread
*t_read
;
158 struct thread
*t_write
;
160 struct restart_info restart
;
163 * For a given daemon, if we've turned on ignore timeouts
164 * ignore the timeout value and assume everything is ok
165 * This is for daemon debugging w/ gdb after we have started
166 * FRR and realize we have something that needs to be looked
172 #define OPTION_MINRESTART 2000
173 #define OPTION_MAXRESTART 2001
174 #define OPTION_DRY 2002
176 static const struct option longopts
[] = {
177 {"daemon", no_argument
, NULL
, 'd'},
178 {"statedir", required_argument
, NULL
, 'S'},
179 {"loglevel", required_argument
, NULL
, 'l'},
180 {"interval", required_argument
, NULL
, 'i'},
181 {"timeout", required_argument
, NULL
, 't'},
182 {"restart-timeout", required_argument
, NULL
, 'T'},
183 {"restart", required_argument
, NULL
, 'r'},
184 {"start-command", required_argument
, NULL
, 's'},
185 {"kill-command", required_argument
, NULL
, 'k'},
186 {"dry", no_argument
, NULL
, OPTION_DRY
},
187 {"min-restart-interval", required_argument
, NULL
, OPTION_MINRESTART
},
188 {"max-restart-interval", required_argument
, NULL
, OPTION_MAXRESTART
},
189 {"pid-file", required_argument
, NULL
, 'p'},
190 {"blank-string", required_argument
, NULL
, 'b'},
191 {"help", no_argument
, NULL
, 'h'},
192 {"version", no_argument
, NULL
, 'v'},
195 static int try_connect(struct daemon
*dmn
);
196 static int wakeup_send_echo(struct thread
*t_wakeup
);
197 static void try_restart(struct daemon
*dmn
);
198 static void phase_check(void);
199 static void restart_done(struct daemon
*dmn
);
201 static const char *progname
;
203 void watchfrr_set_ignore_daemon(struct vty
*vty
, const char *dname
, bool ignore
)
207 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
208 if (strncmp(dmn
->name
, dname
, strlen(dmn
->name
)) == 0)
213 dmn
->ignore_timeout
= ignore
;
214 vty_out(vty
, "%s switching to %s\n", dmn
->name
,
215 ignore
? "ignore" : "watch");
217 vty_out(vty
, "%s is not configured for running at the moment",
221 static void printhelp(FILE *target
)
224 "Usage : %s [OPTION...] <daemon name> ...\n\n\
225 Watchdog program to monitor status of frr daemons and try to restart\n\
226 them if they are down or unresponsive. It determines whether a daemon is\n\
227 up based on whether it can connect to the daemon's vty unix stream socket.\n\
228 It then repeatedly sends echo commands over that socket to determine whether\n\
229 the daemon is responsive. If the daemon crashes, we will receive an EOF\n\
230 on the socket connection and know immediately that the daemon is down.\n\n\
231 The daemons to be monitored should be listed on the command line.\n\n\
232 In order to avoid attempting to restart the daemons in a fast loop,\n\
233 the -m and -M options allow you to control the minimum delay between\n\
234 restart commands. The minimum restart delay is recalculated each time\n\
235 a restart is attempted: if the time since the last restart attempt exceeds\n\
236 twice the -M value, then the restart delay is set to the -m value.\n\
237 Otherwise, the interval is doubled (but capped at the -M value).\n\n",
242 -d, --daemon Run in daemon mode. In this mode, error messages are sent\n\
243 to syslog instead of stdout.\n\
244 -S, --statedir Set the vty socket directory (default is %s)\n\
245 -l, --loglevel Set the logging level (default is %d).\n\
246 The value should range from %d (LOG_EMERG) to %d (LOG_DEBUG),\n\
247 but it can be set higher than %d if extra-verbose debugging\n\
248 messages are desired.\n\
249 --min-restart-interval\n\
250 Set the minimum seconds to wait between invocations of daemon\n\
251 restart commands (default is %d).\n\
252 --max-restart-interval\n\
253 Set the maximum seconds to wait between invocations of daemon\n\
254 restart commands (default is %d).\n\
255 -i, --interval Set the status polling interval in seconds (default is %d)\n\
256 -t, --timeout Set the unresponsiveness timeout in seconds (default is %d)\n\
257 -T, --restart-timeout\n\
258 Set the restart (kill) timeout in seconds (default is %d).\n\
259 If any background jobs are still running after this much\n\
260 time has elapsed, they will be killed.\n\
261 -r, --restart Supply a Bourne shell command to use to restart a single\n\
262 daemon. The command string should include '%%s' where the\n\
263 name of the daemon should be substituted.\n\
265 -s, --start-command\n\
266 Supply a Bourne shell to command to use to start a single\n\
267 daemon. The command string should include '%%s' where the\n\
268 name of the daemon should be substituted.\n\
270 -k, --kill-command\n\
271 Supply a Bourne shell to command to use to stop a single\n\
272 daemon. The command string should include '%%s' where the\n\
273 name of the daemon should be substituted.\n\
275 --dry Do not start or restart anything, just log.\n\
276 -p, --pid-file Set process identifier file name\n\
277 (default is %s/watchfrr.pid).\n\
278 -b, --blank-string\n\
279 When the supplied argument string is found in any of the\n\
280 various shell command arguments (-r, -s, or -k), replace\n\
281 it with a space. This is an ugly hack to circumvent problems\n\
282 passing command-line arguments with embedded spaces.\n\
283 -v, --version Print program version\n\
284 -h, --help Display this help and exit\n",
285 frr_vtydir
, DEFAULT_LOGLEVEL
, LOG_EMERG
, LOG_DEBUG
, LOG_DEBUG
,
286 DEFAULT_MIN_RESTART
, DEFAULT_MAX_RESTART
, DEFAULT_PERIOD
,
287 DEFAULT_TIMEOUT
, DEFAULT_RESTART_TIMEOUT
,
288 DEFAULT_RESTART_CMD
, DEFAULT_START_CMD
, DEFAULT_STOP_CMD
,
292 static pid_t
run_background(char *shell_cmd
)
296 switch (child
= fork()) {
298 flog_err_sys(EC_LIB_SYSTEM_CALL
,
299 "fork failed, cannot run command [%s]: %s",
300 shell_cmd
, safe_strerror(errno
));
304 /* Use separate process group so child processes can be killed
306 if (setpgid(0, 0) < 0)
307 zlog_warn("warning: setpgid(0,0) failed: %s",
308 safe_strerror(errno
));
312 char *const argv
[4] = {shell
, dashc
, shell_cmd
, NULL
};
313 execv("/bin/sh", argv
);
314 flog_err_sys(EC_LIB_SYSTEM_CALL
,
315 "execv(/bin/sh -c '%s') failed: %s",
316 shell_cmd
, safe_strerror(errno
));
320 /* Parent process: we will reap the child later. */
321 zlog_info("Forked background command [pid %d]: %s", (int)child
,
327 static struct timeval
*time_elapsed(struct timeval
*result
,
328 const struct timeval
*start_time
)
330 gettimeofday(result
, NULL
);
331 result
->tv_sec
-= start_time
->tv_sec
;
332 result
->tv_usec
-= start_time
->tv_usec
;
333 while (result
->tv_usec
< 0) {
334 result
->tv_usec
+= 1000000L;
340 static int restart_kill(struct thread
*t_kill
)
342 struct restart_info
*restart
= THREAD_ARG(t_kill
);
343 struct timeval delay
;
345 time_elapsed(&delay
, &restart
->time
);
347 "Warning: %s %s child process %d still running after "
348 "%ld seconds, sending signal %d",
349 restart
->what
, restart
->name
, (int)restart
->pid
,
350 (long)delay
.tv_sec
, (restart
->kills
? SIGKILL
: SIGTERM
));
351 kill(-restart
->pid
, (restart
->kills
? SIGKILL
: SIGTERM
));
353 restart
->t_kill
= NULL
;
354 thread_add_timer(master
, restart_kill
, restart
, gs
.restart_timeout
,
359 static struct restart_info
*find_child(pid_t child
)
362 if (gs
.restart
.pid
== child
)
365 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
366 if (dmn
->restart
.pid
== child
)
367 return &dmn
->restart
;
372 static void sigchild(void)
378 struct restart_info
*restart
;
381 switch (child
= waitpid(-1, &status
, WNOHANG
)) {
383 flog_err_sys(EC_LIB_SYSTEM_CALL
, "waitpid failed: %s",
384 safe_strerror(errno
));
387 zlog_warn("SIGCHLD received, but waitpid did not reap a child");
391 if (child
== integrated_write_pid
) {
392 integrated_write_sigchld(status
);
396 if ((restart
= find_child(child
)) != NULL
) {
397 name
= restart
->name
;
398 what
= restart
->what
;
401 thread_cancel(restart
->t_kill
);
402 restart
->t_kill
= NULL
;
403 /* Update restart time to reflect the time the command
405 gettimeofday(&restart
->time
, NULL
);
409 "waitpid returned status for an unknown child process %d",
414 if (WIFSTOPPED(status
))
415 zlog_warn("warning: %s %s process %d is stopped", what
, name
,
417 else if (WIFSIGNALED(status
))
418 zlog_warn("%s %s process %d terminated due to signal %d", what
,
419 name
, (int)child
, WTERMSIG(status
));
420 else if (WIFEXITED(status
)) {
421 if (WEXITSTATUS(status
) != 0)
423 "%s %s process %d exited with non-zero status %d",
424 what
, name
, (int)child
, WEXITSTATUS(status
));
426 zlog_debug("%s %s process %d exited normally", what
,
429 if (restart
&& restart
!= &gs
.restart
) {
430 dmn
= container_of(restart
, struct daemon
,
434 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
440 "cannot interpret %s %s process %d wait status 0x%x",
441 what
, name
, (int)child
, status
);
445 static int run_job(struct restart_info
*restart
, const char *cmdtype
,
446 const char *command
, int force
, int update_interval
)
448 struct timeval delay
;
450 if (gs
.loglevel
> LOG_DEBUG
+ 1)
451 zlog_debug("attempting to %s %s", cmdtype
, restart
->name
);
454 if (gs
.loglevel
> LOG_DEBUG
+ 1)
456 "cannot %s %s, previous pid %d still running",
457 cmdtype
, restart
->name
, (int)restart
->pid
);
461 #if defined HAVE_SYSTEMD
464 snprintf(buffer
, sizeof(buffer
), "restarting %s", restart
->name
);
465 systemd_send_status(buffer
);
468 /* Note: time_elapsed test must come before the force test, since we
470 to make sure that delay is initialized for use below in updating the
472 if ((time_elapsed(&delay
, &restart
->time
)->tv_sec
< restart
->interval
)
475 if (gs
.loglevel
> LOG_DEBUG
+ 1)
478 "elapsed time %ld < retry interval %ld",
479 cmdtype
, restart
->name
, (long)delay
.tv_sec
,
484 gettimeofday(&restart
->time
, NULL
);
487 char cmd
[strlen(command
) + strlen(restart
->name
) + 1];
488 snprintf(cmd
, sizeof(cmd
), command
, restart
->name
);
489 if ((restart
->pid
= run_background(cmd
)) > 0) {
490 restart
->t_kill
= NULL
;
491 thread_add_timer(master
, restart_kill
, restart
,
492 gs
.restart_timeout
, &restart
->t_kill
);
493 restart
->what
= cmdtype
;
499 #if defined HAVE_SYSTEMD
500 systemd_send_status("FRR Operational");
502 /* Calculate the new restart interval. */
503 if (update_interval
) {
504 if (delay
.tv_sec
> 2 * gs
.max_restart_interval
)
505 restart
->interval
= gs
.min_restart_interval
;
506 else if ((restart
->interval
*= 2) > gs
.max_restart_interval
)
507 restart
->interval
= gs
.max_restart_interval
;
508 if (gs
.loglevel
> LOG_DEBUG
+ 1)
509 zlog_debug("restart %s interval is now %ld",
510 restart
->name
, restart
->interval
);
515 #define SET_READ_HANDLER(DMN) \
517 (DMN)->t_read = NULL; \
518 thread_add_read(master, handle_read, (DMN), (DMN)->fd, \
522 #define SET_WAKEUP_DOWN(DMN) \
524 (DMN)->t_wakeup = NULL; \
525 thread_add_timer_msec(master, wakeup_down, (DMN), \
526 FUZZY(gs.period), &(DMN)->t_wakeup); \
529 #define SET_WAKEUP_UNRESPONSIVE(DMN) \
531 (DMN)->t_wakeup = NULL; \
532 thread_add_timer_msec(master, wakeup_unresponsive, (DMN), \
533 FUZZY(gs.period), &(DMN)->t_wakeup); \
536 #define SET_WAKEUP_ECHO(DMN) \
538 (DMN)->t_wakeup = NULL; \
539 thread_add_timer_msec(master, wakeup_send_echo, (DMN), \
540 FUZZY(gs.period), &(DMN)->t_wakeup); \
543 static int wakeup_down(struct thread
*t_wakeup
)
545 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
547 dmn
->t_wakeup
= NULL
;
548 if (try_connect(dmn
) < 0)
549 SET_WAKEUP_DOWN(dmn
);
550 if ((dmn
->connect_tries
> 1) && (dmn
->state
!= DAEMON_UP
))
555 static int wakeup_init(struct thread
*t_wakeup
)
557 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
559 dmn
->t_wakeup
= NULL
;
560 if (try_connect(dmn
) < 0) {
562 "%s state -> down : initial connection attempt failed",
564 dmn
->state
= DAEMON_DOWN
;
570 static void restart_done(struct daemon
*dmn
)
572 if (dmn
->state
!= DAEMON_DOWN
) {
574 "Daemon: %s: is in %s state but expected it to be in DAEMON_DOWN state",
575 dmn
->name
, state_str
[dmn
->state
]);
579 THREAD_OFF(dmn
->t_wakeup
);
580 if (try_connect(dmn
) < 0)
581 SET_WAKEUP_DOWN(dmn
);
584 static void daemon_down(struct daemon
*dmn
, const char *why
)
586 if (IS_UP(dmn
) || (dmn
->state
== DAEMON_INIT
))
587 flog_err(EC_WATCHFRR_CONNECTION
, "%s state -> down : %s",
589 else if (gs
.loglevel
> LOG_DEBUG
)
590 zlog_debug("%s still down : %s", dmn
->name
, why
);
593 dmn
->state
= DAEMON_DOWN
;
598 THREAD_OFF(dmn
->t_read
);
599 THREAD_OFF(dmn
->t_write
);
600 THREAD_OFF(dmn
->t_wakeup
);
601 if (try_connect(dmn
) < 0)
602 SET_WAKEUP_DOWN(dmn
);
606 static int handle_read(struct thread
*t_read
)
608 struct daemon
*dmn
= THREAD_ARG(t_read
);
609 static const char resp
[sizeof(PING_TOKEN
) + 4] = PING_TOKEN
"\n";
610 char buf
[sizeof(resp
) + 100];
612 struct timeval delay
;
615 if ((rc
= read(dmn
->fd
, buf
, sizeof(buf
))) < 0) {
618 if (ERRNO_IO_RETRY(errno
)) {
619 /* Pretend it never happened. */
620 SET_READ_HANDLER(dmn
);
623 snprintf(why
, sizeof(why
), "unexpected read error: %s",
624 safe_strerror(errno
));
625 daemon_down(dmn
, why
);
629 daemon_down(dmn
, "read returned EOF");
632 if (!dmn
->echo_sent
.tv_sec
) {
633 char why
[sizeof(buf
) + 100];
634 snprintf(why
, sizeof(why
),
635 "unexpected read returns %d bytes: %.*s", (int)rc
,
637 daemon_down(dmn
, why
);
641 /* We are expecting an echo response: is there any chance that the
642 response would not be returned entirely in the first read? That
643 seems inconceivable... */
644 if ((rc
!= sizeof(resp
)) || memcmp(buf
, resp
, sizeof(resp
))) {
645 char why
[100 + sizeof(buf
)];
646 snprintf(why
, sizeof(why
),
647 "read returned bad echo response of %d bytes "
648 "(expecting %u): %.*s",
649 (int)rc
, (unsigned int)sizeof(resp
), (int)rc
, buf
);
650 daemon_down(dmn
, why
);
654 time_elapsed(&delay
, &dmn
->echo_sent
);
655 dmn
->echo_sent
.tv_sec
= 0;
656 if (dmn
->state
== DAEMON_UNRESPONSIVE
) {
657 if (delay
.tv_sec
< gs
.timeout
) {
658 dmn
->state
= DAEMON_UP
;
660 "%s state -> up : echo response received after %ld.%06ld "
662 dmn
->name
, (long)delay
.tv_sec
,
663 (long)delay
.tv_usec
);
666 "%s: slow echo response finally received after %ld.%06ld "
668 dmn
->name
, (long)delay
.tv_sec
,
669 (long)delay
.tv_usec
);
670 } else if (gs
.loglevel
> LOG_DEBUG
+ 1)
671 zlog_debug("%s: echo response received after %ld.%06ld seconds",
672 dmn
->name
, (long)delay
.tv_sec
, (long)delay
.tv_usec
);
674 SET_READ_HANDLER(dmn
);
676 thread_cancel(dmn
->t_wakeup
);
677 SET_WAKEUP_ECHO(dmn
);
683 * Wait till we notice that all daemons are ready before
684 * we send we are ready to systemd
686 static void daemon_send_ready(int exitcode
)
696 zlog_notice("all daemons up, doing startup-complete notify");
697 else if (gs
.numdown
< gs
.numdaemons
)
698 flog_err(EC_WATCHFRR_CONNECTION
,
699 "startup did not complete within timeout"
700 " (%d/%d daemons running)",
701 gs
.numdaemons
- gs
.numdown
, gs
.numdaemons
);
703 flog_err(EC_WATCHFRR_CONNECTION
,
704 "all configured daemons failed to start"
705 " -- exiting watchfrr");
712 snprintf(started
, sizeof(started
), "%s%s", frr_vtydir
,
714 fp
= fopen(started
, "w");
717 #if defined HAVE_SYSTEMD
718 systemd_send_started(master
, 0);
719 systemd_send_status("FRR Operational");
724 static void daemon_up(struct daemon
*dmn
, const char *why
)
726 dmn
->state
= DAEMON_UP
;
728 dmn
->connect_tries
= 0;
729 zlog_notice("%s state -> up : %s", dmn
->name
, why
);
731 daemon_send_ready(0);
732 SET_WAKEUP_ECHO(dmn
);
736 static int check_connect(struct thread
*t_write
)
738 struct daemon
*dmn
= THREAD_ARG(t_write
);
740 socklen_t reslen
= sizeof(sockerr
);
743 if (getsockopt(dmn
->fd
, SOL_SOCKET
, SO_ERROR
, (char *)&sockerr
, &reslen
)
745 zlog_warn("%s: check_connect: getsockopt failed: %s", dmn
->name
,
746 safe_strerror(errno
));
748 "getsockopt failed checking connection success");
751 if ((reslen
== sizeof(sockerr
)) && sockerr
) {
755 "getsockopt reports that connection attempt failed: %s",
756 safe_strerror(sockerr
));
757 daemon_down(dmn
, why
);
761 daemon_up(dmn
, "delayed connect succeeded");
765 static int wakeup_connect_hanging(struct thread
*t_wakeup
)
767 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
770 dmn
->t_wakeup
= NULL
;
771 snprintf(why
, sizeof(why
),
772 "connection attempt timed out after %ld seconds", gs
.timeout
);
773 daemon_down(dmn
, why
);
777 /* Making connection to protocol daemon. */
778 static int try_connect(struct daemon
*dmn
)
781 struct sockaddr_un addr
;
784 if (gs
.loglevel
> LOG_DEBUG
+ 1)
785 zlog_debug("%s: attempting to connect", dmn
->name
);
786 dmn
->connect_tries
++;
788 memset(&addr
, 0, sizeof(struct sockaddr_un
));
789 addr
.sun_family
= AF_UNIX
;
790 snprintf(addr
.sun_path
, sizeof(addr
.sun_path
), "%s/%s.vty", gs
.vtydir
,
792 #ifdef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN
793 len
= addr
.sun_len
= SUN_LEN(&addr
);
795 len
= sizeof(addr
.sun_family
) + strlen(addr
.sun_path
);
796 #endif /* HAVE_STRUCT_SOCKADDR_UN_SUN_LEN */
798 /* Quick check to see if we might succeed before we go to the trouble
799 of creating a socket. */
800 if (access(addr
.sun_path
, W_OK
) < 0) {
802 flog_err_sys(EC_LIB_SYSTEM_CALL
,
803 "%s: access to socket %s denied: %s",
804 dmn
->name
, addr
.sun_path
,
805 safe_strerror(errno
));
809 if ((sock
= socket(AF_UNIX
, SOCK_STREAM
, 0)) < 0) {
810 flog_err_sys(EC_LIB_SOCKET
, "%s(%s): cannot make socket: %s",
811 __func__
, addr
.sun_path
, safe_strerror(errno
));
815 if (set_nonblocking(sock
) < 0 || set_cloexec(sock
) < 0) {
816 flog_err_sys(EC_LIB_SYSTEM_CALL
,
817 "%s(%s): set_nonblocking/cloexec(%d) failed",
818 __func__
, addr
.sun_path
, sock
);
823 if (connect(sock
, (struct sockaddr
*)&addr
, len
) < 0) {
824 if ((errno
!= EINPROGRESS
) && (errno
!= EWOULDBLOCK
)) {
825 if (gs
.loglevel
> LOG_DEBUG
)
826 zlog_debug("%s(%s): connect failed: %s",
827 __func__
, addr
.sun_path
,
828 safe_strerror(errno
));
832 if (gs
.loglevel
> LOG_DEBUG
)
833 zlog_debug("%s: connection in progress", dmn
->name
);
834 dmn
->state
= DAEMON_CONNECTING
;
837 thread_add_write(master
, check_connect
, dmn
, dmn
->fd
,
839 dmn
->t_wakeup
= NULL
;
840 thread_add_timer(master
, wakeup_connect_hanging
, dmn
,
841 gs
.timeout
, &dmn
->t_wakeup
);
842 SET_READ_HANDLER(dmn
);
847 SET_READ_HANDLER(dmn
);
848 daemon_up(dmn
, "connect succeeded");
852 static int phase_hanging(struct thread
*t_hanging
)
854 gs
.t_phase_hanging
= NULL
;
855 flog_err(EC_WATCHFRR_CONNECTION
,
856 "Phase [%s] hanging for %ld seconds, aborting phased restart",
857 phase_str
[gs
.phase
], PHASE_TIMEOUT
);
858 gs
.phase
= PHASE_NONE
;
862 static void set_phase(restart_phase_t new_phase
)
864 gs
.phase
= new_phase
;
865 if (gs
.t_phase_hanging
)
866 thread_cancel(gs
.t_phase_hanging
);
867 gs
.t_phase_hanging
= NULL
;
868 thread_add_timer(master
, phase_hanging
, NULL
, PHASE_TIMEOUT
,
869 &gs
.t_phase_hanging
);
872 static void phase_check(void)
881 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
882 if (dmn
->state
== DAEMON_INIT
)
885 /* startup complete, everything out of INIT */
886 gs
.phase
= PHASE_NONE
;
887 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
888 if (dmn
->state
== DAEMON_DOWN
) {
889 SET_WAKEUP_DOWN(dmn
);
893 case PHASE_STOPS_PENDING
:
897 "Phased restart: all routing daemon stop jobs have completed.");
898 set_phase(PHASE_WAITING_DOWN
);
901 case PHASE_WAITING_DOWN
:
902 if (gs
.numdown
+ IS_UP(gs
.special
) < gs
.numdaemons
)
904 zlog_info("Phased restart: all routing daemons now down.");
905 run_job(&gs
.special
->restart
, "restart", gs
.restart_command
, 1,
907 set_phase(PHASE_ZEBRA_RESTART_PENDING
);
910 case PHASE_ZEBRA_RESTART_PENDING
:
911 if (gs
.special
->restart
.pid
)
913 zlog_info("Phased restart: %s restart job completed.",
915 set_phase(PHASE_WAITING_ZEBRA_UP
);
918 case PHASE_WAITING_ZEBRA_UP
:
919 if (!IS_UP(gs
.special
))
921 zlog_info("Phased restart: %s is now up.", gs
.special
->name
);
924 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
925 if (dmn
!= gs
.special
)
926 run_job(&dmn
->restart
, "start",
927 gs
.start_command
, 1, 0);
930 gs
.phase
= PHASE_NONE
;
931 THREAD_OFF(gs
.t_phase_hanging
);
932 zlog_notice("Phased global restart has completed.");
937 static void try_restart(struct daemon
*dmn
)
942 if (dmn
!= gs
.special
) {
943 if ((gs
.special
->state
== DAEMON_UP
)
944 && (gs
.phase
== PHASE_NONE
))
945 run_job(&dmn
->restart
, "restart", gs
.restart_command
, 0,
949 "%s: postponing restart attempt because master %s daemon "
950 "not up [%s], or phased restart in progress",
951 dmn
->name
, gs
.special
->name
,
952 state_str
[gs
.special
->state
]);
956 if ((gs
.phase
!= PHASE_NONE
) || gs
.numpids
) {
957 if (gs
.loglevel
> LOG_DEBUG
+ 1)
959 "postponing phased global restart: restart already in "
960 "progress [%s], or outstanding child processes [%d]",
961 phase_str
[gs
.phase
], gs
.numpids
);
964 /* Is it too soon for a restart? */
966 struct timeval delay
;
967 if (time_elapsed(&delay
, &gs
.special
->restart
.time
)->tv_sec
968 < gs
.special
->restart
.interval
) {
969 if (gs
.loglevel
> LOG_DEBUG
+ 1)
971 "postponing phased global restart: "
972 "elapsed time %ld < retry interval %ld",
974 gs
.special
->restart
.interval
);
978 run_job(&gs
.restart
, "restart", gs
.restart_command
, 0, 1);
981 static int wakeup_unresponsive(struct thread
*t_wakeup
)
983 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
985 dmn
->t_wakeup
= NULL
;
986 if (dmn
->state
!= DAEMON_UNRESPONSIVE
)
987 flog_err(EC_WATCHFRR_CONNECTION
,
988 "%s: no longer unresponsive (now %s), "
989 "wakeup should have been cancelled!",
990 dmn
->name
, state_str
[dmn
->state
]);
992 SET_WAKEUP_UNRESPONSIVE(dmn
);
998 static int wakeup_no_answer(struct thread
*t_wakeup
)
1000 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
1002 dmn
->t_wakeup
= NULL
;
1003 dmn
->state
= DAEMON_UNRESPONSIVE
;
1004 if (dmn
->ignore_timeout
)
1006 flog_err(EC_WATCHFRR_CONNECTION
,
1007 "%s state -> unresponsive : no response yet to ping "
1008 "sent %ld seconds ago",
1009 dmn
->name
, gs
.timeout
);
1010 SET_WAKEUP_UNRESPONSIVE(dmn
);
1015 static int wakeup_send_echo(struct thread
*t_wakeup
)
1017 static const char echocmd
[] = "echo " PING_TOKEN
;
1019 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
1021 dmn
->t_wakeup
= NULL
;
1022 if (((rc
= write(dmn
->fd
, echocmd
, sizeof(echocmd
))) < 0)
1023 || ((size_t)rc
!= sizeof(echocmd
))) {
1024 char why
[100 + sizeof(echocmd
)];
1025 snprintf(why
, sizeof(why
),
1026 "write '%s' returned %d instead of %u", echocmd
,
1027 (int)rc
, (unsigned int)sizeof(echocmd
));
1028 daemon_down(dmn
, why
);
1030 gettimeofday(&dmn
->echo_sent
, NULL
);
1031 dmn
->t_wakeup
= NULL
;
1032 thread_add_timer(master
, wakeup_no_answer
, dmn
, gs
.timeout
,
1038 bool check_all_up(void)
1042 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
1043 if (dmn
->state
!= DAEMON_UP
)
1048 void watchfrr_status(struct vty
*vty
)
1051 struct timeval delay
;
1053 vty_out(vty
, "watchfrr global phase: %s\n", phase_str
[gs
.phase
]);
1055 vty_out(vty
, " global restart running, pid %ld\n",
1056 (long)gs
.restart
.pid
);
1058 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
1059 vty_out(vty
, " %-20s %s%s", dmn
->name
, state_str
[dmn
->state
],
1060 dmn
->ignore_timeout
? "/Ignoring Timeout\n" : "\n");
1061 if (dmn
->restart
.pid
)
1062 vty_out(vty
, " restart running, pid %ld\n",
1063 (long)dmn
->restart
.pid
);
1064 else if (dmn
->state
== DAEMON_DOWN
&&
1065 time_elapsed(&delay
, &dmn
->restart
.time
)->tv_sec
1066 < dmn
->restart
.interval
)
1067 vty_out(vty
, " restarting in %jd seconds"
1068 " (%jds backoff interval)\n",
1069 (intmax_t)dmn
->restart
.interval
1070 - (intmax_t)delay
.tv_sec
,
1071 (intmax_t)dmn
->restart
.interval
);
1075 static void sigint(void)
1077 zlog_notice("Terminating on signal");
1078 systemd_send_stopping();
1082 static int valid_command(const char *cmd
)
1086 return ((p
= strchr(cmd
, '%')) != NULL
) && (*(p
+ 1) == 's')
1087 && !strchr(p
+ 1, '%');
1090 /* This is an ugly hack to circumvent problems with passing command-line
1091 arguments that contain spaces. The fix is to use a configuration file. */
1092 static char *translate_blanks(const char *cmd
, const char *blankstr
)
1096 size_t bslen
= strlen(blankstr
);
1098 if (!(res
= strdup(cmd
))) {
1102 while ((p
= strstr(res
, blankstr
)) != NULL
) {
1105 memmove(p
+ 1, p
+ bslen
, strlen(p
+ bslen
) + 1);
1110 static int startup_timeout(struct thread
*t_wakeup
)
1112 daemon_send_ready(1);
1116 static void watchfrr_init(int argc
, char **argv
)
1118 const char *special
= "zebra";
1120 struct daemon
*dmn
, **add
= &gs
.daemons
;
1121 char alldaemons
[512] = "", *p
= alldaemons
;
1123 thread_add_timer_msec(master
, startup_timeout
, NULL
, STARTUP_TIMEOUT
,
1124 &gs
.t_startup_timeout
);
1126 for (i
= optind
; i
< argc
; i
++) {
1127 dmn
= XCALLOC(MTYPE_WATCHFRR_DAEMON
, sizeof(*dmn
));
1129 dmn
->name
= dmn
->restart
.name
= argv
[i
];
1130 dmn
->state
= DAEMON_INIT
;
1134 dmn
->t_wakeup
= NULL
;
1135 thread_add_timer_msec(master
, wakeup_init
, dmn
, 0,
1137 dmn
->restart
.interval
= gs
.min_restart_interval
;
1141 if (!strcmp(dmn
->name
, special
))
1147 "Must specify one or more daemons to monitor.\n\n");
1150 if (!watch_only
&& !gs
.special
) {
1151 fprintf(stderr
, "\"%s\" daemon must be in daemon lists\n\n",
1156 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
1157 snprintf(p
, alldaemons
+ sizeof(alldaemons
) - p
, "%s%s",
1158 (p
== alldaemons
) ? "" : " ", dmn
->name
);
1161 zlog_notice("%s %s watching [%s]%s", progname
, FRR_VERSION
, alldaemons
,
1162 watch_only
? ", monitor mode" : "");
1165 struct zebra_privs_t watchfrr_privs
= {
1167 .vty_group
= VTY_GROUP
,
1171 static struct quagga_signal_t watchfrr_signals
[] = {
1182 .handler
= sigchild
,
1186 FRR_DAEMON_INFO(watchfrr
, WATCHFRR
,
1187 .flags
= FRR_NO_PRIVSEP
| FRR_NO_TCPVTY
| FRR_LIMITED_CLI
1188 | FRR_NO_CFG_PID_DRY
| FRR_NO_ZCLIENT
1191 .printhelp
= printhelp
,
1192 .copyright
= "Copyright 2004 Andrew J. Schorr",
1194 .signals
= watchfrr_signals
,
1195 .n_signals
= array_size(watchfrr_signals
),
1197 .privs
= &watchfrr_privs
, )
1199 #define DEPRECATED_OPTIONS "aAezR:"
1201 int main(int argc
, char **argv
)
1204 const char *blankstr
= NULL
;
1206 frr_preinit(&watchfrr_di
, argc
, argv
);
1207 progname
= watchfrr_di
.progname
;
1209 frr_opt_add("b:dk:l:i:p:r:S:s:t:T:" DEPRECATED_OPTIONS
, longopts
, "");
1211 gs
.restart
.name
= "all";
1212 while ((opt
= frr_getopt(argc
, argv
, NULL
)) != EOF
) {
1213 if (opt
&& opt
< 128 && strchr(DEPRECATED_OPTIONS
, opt
)) {
1215 "The -%c option no longer exists.\n"
1216 "Please refer to the watchfrr(8) man page.\n",
1231 if (!valid_command(optarg
)) {
1233 "Invalid kill command, must contain '%%s': %s\n",
1237 gs
.stop_command
= optarg
;
1241 if ((sscanf(optarg
, "%d%1s", &gs
.loglevel
, garbage
)
1243 || (gs
.loglevel
< LOG_EMERG
)) {
1245 "Invalid loglevel argument: %s\n",
1250 case OPTION_MINRESTART
: {
1252 if ((sscanf(optarg
, "%ld%1s", &gs
.min_restart_interval
,
1255 || (gs
.min_restart_interval
< 0)) {
1257 "Invalid min_restart_interval argument: %s\n",
1262 case OPTION_MAXRESTART
: {
1264 if ((sscanf(optarg
, "%ld%1s", &gs
.max_restart_interval
,
1267 || (gs
.max_restart_interval
< 0)) {
1269 "Invalid max_restart_interval argument: %s\n",
1277 if ((sscanf(optarg
, "%d%1s", &period
, garbage
) != 1)
1278 || (gs
.period
< 1)) {
1280 "Invalid interval argument: %s\n",
1284 gs
.period
= 1000 * period
;
1287 watchfrr_di
.pid_file
= optarg
;
1290 if (!valid_command(optarg
)) {
1292 "Invalid restart command, must contain '%%s': %s\n",
1296 gs
.restart_command
= optarg
;
1299 if (!valid_command(optarg
)) {
1301 "Invalid start command, must contain '%%s': %s\n",
1305 gs
.start_command
= optarg
;
1312 if ((sscanf(optarg
, "%ld%1s", &gs
.timeout
, garbage
)
1314 || (gs
.timeout
< 1)) {
1316 "Invalid timeout argument: %s\n",
1323 if ((sscanf(optarg
, "%ld%1s", &gs
.restart_timeout
,
1326 || (gs
.restart_timeout
< 1)) {
1328 "Invalid restart timeout argument: %s\n",
1334 fputs("Invalid option.\n", stderr
);
1340 && (gs
.start_command
|| gs
.stop_command
|| gs
.restart_command
)) {
1341 fputs("Options -r/-s/-k are not used when --dry is active.\n",
1345 && (!gs
.restart_command
|| !gs
.start_command
|| !gs
.stop_command
)) {
1347 "Options -s (start), -k (kill), and -r (restart) are required.\n");
1352 if (gs
.restart_command
)
1353 gs
.restart_command
=
1354 translate_blanks(gs
.restart_command
, blankstr
);
1355 if (gs
.start_command
)
1357 translate_blanks(gs
.start_command
, blankstr
);
1358 if (gs
.stop_command
)
1360 translate_blanks(gs
.stop_command
, blankstr
);
1363 gs
.restart
.interval
= gs
.min_restart_interval
;
1365 master
= frr_init();
1366 watchfrr_error_init();
1367 watchfrr_init(argc
, argv
);
1368 watchfrr_vty_init();
1372 zlog_set_level(ZLOG_DEST_MONITOR
, ZLOG_DISABLED
);
1373 if (watchfrr_di
.daemon_mode
)
1374 zlog_set_level(ZLOG_DEST_SYSLOG
, MIN(gs
.loglevel
, LOG_DEBUG
));
1376 zlog_set_level(ZLOG_DEST_STDOUT
, MIN(gs
.loglevel
, LOG_DEBUG
));
1380 systemd_send_stopping();