2 * Monitor status of frr daemons and restart if necessary.
4 * Copyright (C) 2004 Andrew J. Schorr
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; see the file COPYING; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26 #include <lib/version.h>
28 #include "memory_vty.h"
30 #include "lib_errors.h"
39 #include "watchfrr_errors.h"
42 #define MIN(X,Y) (((X) <= (Y)) ? (X) : (Y))
45 /* Macros to help randomize timers. */
46 #define JITTER(X) ((random() % ((X)+1))-((X)/2))
47 #define FUZZY(X) ((X)+JITTER((X)/20))
49 #define DEFAULT_PERIOD 5
50 #define DEFAULT_TIMEOUT 90
51 #define DEFAULT_RESTART_TIMEOUT 20
52 #define DEFAULT_LOGLEVEL LOG_INFO
53 #define DEFAULT_MIN_RESTART 60
54 #define DEFAULT_MAX_RESTART 600
56 #define PING_TOKEN "PING"
58 /* Needs to be global, referenced somewhere inside libfrr. */
59 struct thread_master
*master
;
60 static char pidfile_default
[256];
62 static bool watch_only
= false;
68 PHASE_ZEBRA_RESTART_PENDING
,
69 PHASE_WAITING_ZEBRA_UP
72 static const char *phase_str
[] = {
75 "Waiting for other daemons to come down",
76 "Zebra restart job running",
77 "Waiting for zebra to come up",
81 #define PHASE_TIMEOUT (3*gs.restart_timeout)
89 struct thread
*t_kill
;
93 static struct global_state
{
94 restart_phase_t phase
;
95 struct thread
*t_phase_hanging
;
100 long min_restart_interval
;
101 long max_restart_interval
;
102 struct daemon
*daemons
;
103 const char *restart_command
;
104 const char *start_command
;
105 const char *stop_command
;
106 struct restart_info restart
;
108 struct daemon
*special
; /* points to zebra when doing phased restart */
111 int numdown
; /* # of daemons that are not UP or UNRESPONSIVE */
114 .vtydir
= frr_vtydir
,
115 .period
= 1000 * DEFAULT_PERIOD
,
116 .timeout
= DEFAULT_TIMEOUT
,
117 .restart_timeout
= DEFAULT_RESTART_TIMEOUT
,
118 .loglevel
= DEFAULT_LOGLEVEL
,
119 .min_restart_interval
= DEFAULT_MIN_RESTART
,
120 .max_restart_interval
= DEFAULT_MAX_RESTART
,
132 (((DMN)->state == DAEMON_UP) || ((DMN)->state == DAEMON_UNRESPONSIVE))
134 static const char *state_str
[] = {
135 "Init", "Down", "Connecting", "Up", "Unresponsive",
140 daemon_state_t state
;
142 struct timeval echo_sent
;
143 unsigned int connect_tries
;
144 struct thread
*t_wakeup
;
145 struct thread
*t_read
;
146 struct thread
*t_write
;
148 struct restart_info restart
;
151 #define OPTION_MINRESTART 2000
152 #define OPTION_MAXRESTART 2001
153 #define OPTION_DRY 2002
155 static const struct option longopts
[] = {
156 {"daemon", no_argument
, NULL
, 'd'},
157 {"statedir", required_argument
, NULL
, 'S'},
158 {"loglevel", required_argument
, NULL
, 'l'},
159 {"interval", required_argument
, NULL
, 'i'},
160 {"timeout", required_argument
, NULL
, 't'},
161 {"restart-timeout", required_argument
, NULL
, 'T'},
162 {"restart", required_argument
, NULL
, 'r'},
163 {"start-command", required_argument
, NULL
, 's'},
164 {"kill-command", required_argument
, NULL
, 'k'},
165 {"dry", no_argument
, NULL
, OPTION_DRY
},
166 {"min-restart-interval", required_argument
, NULL
, OPTION_MINRESTART
},
167 {"max-restart-interval", required_argument
, NULL
, OPTION_MAXRESTART
},
168 {"pid-file", required_argument
, NULL
, 'p'},
169 {"blank-string", required_argument
, NULL
, 'b'},
170 {"help", no_argument
, NULL
, 'h'},
171 {"version", no_argument
, NULL
, 'v'},
174 static int try_connect(struct daemon
*dmn
);
175 static int wakeup_send_echo(struct thread
*t_wakeup
);
176 static void try_restart(struct daemon
*dmn
);
177 static void phase_check(void);
179 static const char *progname
;
180 static void printhelp(FILE *target
)
183 "Usage : %s [OPTION...] <daemon name> ...\n\n\
184 Watchdog program to monitor status of frr daemons and try to restart\n\
185 them if they are down or unresponsive. It determines whether a daemon is\n\
186 up based on whether it can connect to the daemon's vty unix stream socket.\n\
187 It then repeatedly sends echo commands over that socket to determine whether\n\
188 the daemon is responsive. If the daemon crashes, we will receive an EOF\n\
189 on the socket connection and know immediately that the daemon is down.\n\n\
190 The daemons to be monitored should be listed on the command line.\n\n\
191 In order to avoid attempting to restart the daemons in a fast loop,\n\
192 the -m and -M options allow you to control the minimum delay between\n\
193 restart commands. The minimum restart delay is recalculated each time\n\
194 a restart is attempted: if the time since the last restart attempt exceeds\n\
195 twice the -M value, then the restart delay is set to the -m value.\n\
196 Otherwise, the interval is doubled (but capped at the -M value).\n\n",
201 -d, --daemon Run in daemon mode. In this mode, error messages are sent\n\
202 to syslog instead of stdout.\n\
203 -S, --statedir Set the vty socket directory (default is %s)\n\
204 -l, --loglevel Set the logging level (default is %d).\n\
205 The value should range from %d (LOG_EMERG) to %d (LOG_DEBUG),\n\
206 but it can be set higher than %d if extra-verbose debugging\n\
207 messages are desired.\n\
208 --min-restart-interval\n\
209 Set the minimum seconds to wait between invocations of daemon\n\
210 restart commands (default is %d).\n\
211 --max-restart-interval\n\
212 Set the maximum seconds to wait between invocations of daemon\n\
213 restart commands (default is %d).\n\
214 -i, --interval Set the status polling interval in seconds (default is %d)\n\
215 -t, --timeout Set the unresponsiveness timeout in seconds (default is %d)\n\
216 -T, --restart-timeout\n\
217 Set the restart (kill) timeout in seconds (default is %d).\n\
218 If any background jobs are still running after this much\n\
219 time has elapsed, they will be killed.\n\
220 -r, --restart Supply a Bourne shell command to use to restart a single\n\
221 daemon. The command string should include '%%s' where the\n\
222 name of the daemon should be substituted.\n\
223 -s, --start-command\n\
224 Supply a Bourne shell to command to use to start a single\n\
225 daemon. The command string should include '%%s' where the\n\
226 name of the daemon should be substituted.\n\
227 -k, --kill-command\n\
228 Supply a Bourne shell to command to use to stop a single\n\
229 daemon. The command string should include '%%s' where the\n\
230 name of the daemon should be substituted.\n\
231 --dry Do not start or restart anything, just log.\n\
232 -p, --pid-file Set process identifier file name\n\
234 -b, --blank-string\n\
235 When the supplied argument string is found in any of the\n\
236 various shell command arguments (-r, -s, or -k), replace\n\
237 it with a space. This is an ugly hack to circumvent problems\n\
238 passing command-line arguments with embedded spaces.\n\
239 -v, --version Print program version\n\
240 -h, --help Display this help and exit\n",
241 frr_vtydir
, DEFAULT_LOGLEVEL
, LOG_EMERG
, LOG_DEBUG
, LOG_DEBUG
,
242 DEFAULT_MIN_RESTART
, DEFAULT_MAX_RESTART
, DEFAULT_PERIOD
,
243 DEFAULT_TIMEOUT
, DEFAULT_RESTART_TIMEOUT
, pidfile_default
);
246 static pid_t
run_background(char *shell_cmd
)
250 switch (child
= fork()) {
252 flog_err_sys(LIB_ERR_SYSTEM_CALL
,
253 "fork failed, cannot run command [%s]: %s",
254 shell_cmd
, safe_strerror(errno
));
258 /* Use separate process group so child processes can be killed
260 if (setpgid(0, 0) < 0)
261 zlog_warn("warning: setpgid(0,0) failed: %s",
262 safe_strerror(errno
));
266 char *const argv
[4] = {shell
, dashc
, shell_cmd
, NULL
};
267 execv("/bin/sh", argv
);
268 flog_err_sys(LIB_ERR_SYSTEM_CALL
,
269 "execv(/bin/sh -c '%s') failed: %s",
270 shell_cmd
, safe_strerror(errno
));
274 /* Parent process: we will reap the child later. */
275 flog_err_sys(LIB_ERR_SYSTEM_CALL
,
276 "Forked background command [pid %d]: %s",
277 (int)child
, shell_cmd
);
282 static struct timeval
*time_elapsed(struct timeval
*result
,
283 const struct timeval
*start_time
)
285 gettimeofday(result
, NULL
);
286 result
->tv_sec
-= start_time
->tv_sec
;
287 result
->tv_usec
-= start_time
->tv_usec
;
288 while (result
->tv_usec
< 0) {
289 result
->tv_usec
+= 1000000L;
295 static int restart_kill(struct thread
*t_kill
)
297 struct restart_info
*restart
= THREAD_ARG(t_kill
);
298 struct timeval delay
;
300 time_elapsed(&delay
, &restart
->time
);
302 "Warning: %s %s child process %d still running after "
303 "%ld seconds, sending signal %d",
304 restart
->what
, restart
->name
, (int)restart
->pid
,
305 (long)delay
.tv_sec
, (restart
->kills
? SIGKILL
: SIGTERM
));
306 kill(-restart
->pid
, (restart
->kills
? SIGKILL
: SIGTERM
));
308 restart
->t_kill
= NULL
;
309 thread_add_timer(master
, restart_kill
, restart
, gs
.restart_timeout
,
314 static struct restart_info
*find_child(pid_t child
)
317 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
318 if (dmn
->restart
.pid
== child
)
319 return &dmn
->restart
;
324 static void sigchild(void)
330 struct restart_info
*restart
;
332 switch (child
= waitpid(-1, &status
, WNOHANG
)) {
334 flog_err_sys(LIB_ERR_SYSTEM_CALL
, "waitpid failed: %s",
335 safe_strerror(errno
));
338 zlog_warn("SIGCHLD received, but waitpid did not reap a child");
342 if (child
== integrated_write_pid
) {
343 integrated_write_sigchld(status
);
347 if ((restart
= find_child(child
)) != NULL
) {
348 name
= restart
->name
;
349 what
= restart
->what
;
352 thread_cancel(restart
->t_kill
);
353 restart
->t_kill
= NULL
;
354 /* Update restart time to reflect the time the command
356 gettimeofday(&restart
->time
, NULL
);
360 "waitpid returned status for an unknown child process %d",
365 if (WIFSTOPPED(status
))
366 zlog_warn("warning: %s %s process %d is stopped", what
, name
,
368 else if (WIFSIGNALED(status
))
369 zlog_warn("%s %s process %d terminated due to signal %d", what
,
370 name
, (int)child
, WTERMSIG(status
));
371 else if (WIFEXITED(status
)) {
372 if (WEXITSTATUS(status
) != 0)
374 "%s %s process %d exited with non-zero status %d",
375 what
, name
, (int)child
, WEXITSTATUS(status
));
377 zlog_debug("%s %s process %d exited normally", what
,
382 "cannot interpret %s %s process %d wait status 0x%x",
383 what
, name
, (int)child
, status
);
387 static int run_job(struct restart_info
*restart
, const char *cmdtype
,
388 const char *command
, int force
, int update_interval
)
390 struct timeval delay
;
392 if (gs
.loglevel
> LOG_DEBUG
+ 1)
393 zlog_debug("attempting to %s %s", cmdtype
, restart
->name
);
396 if (gs
.loglevel
> LOG_DEBUG
+ 1)
398 "cannot %s %s, previous pid %d still running",
399 cmdtype
, restart
->name
, (int)restart
->pid
);
403 /* Note: time_elapsed test must come before the force test, since we
405 to make sure that delay is initialized for use below in updating the
407 if ((time_elapsed(&delay
, &restart
->time
)->tv_sec
< restart
->interval
)
409 if (gs
.loglevel
> LOG_DEBUG
+ 1)
412 "elapsed time %ld < retry interval %ld",
413 cmdtype
, restart
->name
, (long)delay
.tv_sec
,
418 gettimeofday(&restart
->time
, NULL
);
421 char cmd
[strlen(command
) + strlen(restart
->name
) + 1];
422 snprintf(cmd
, sizeof(cmd
), command
, restart
->name
);
423 if ((restart
->pid
= run_background(cmd
)) > 0) {
424 restart
->t_kill
= NULL
;
425 thread_add_timer(master
, restart_kill
, restart
,
426 gs
.restart_timeout
, &restart
->t_kill
);
427 restart
->what
= cmdtype
;
433 /* Calculate the new restart interval. */
434 if (update_interval
) {
435 if (delay
.tv_sec
> 2 * gs
.max_restart_interval
)
436 restart
->interval
= gs
.min_restart_interval
;
437 else if ((restart
->interval
*= 2) > gs
.max_restart_interval
)
438 restart
->interval
= gs
.max_restart_interval
;
439 if (gs
.loglevel
> LOG_DEBUG
+ 1)
440 zlog_debug("restart %s interval is now %ld",
441 restart
->name
, restart
->interval
);
446 #define SET_READ_HANDLER(DMN) \
448 (DMN)->t_read = NULL; \
449 thread_add_read(master, handle_read, (DMN), (DMN)->fd, \
453 #define SET_WAKEUP_DOWN(DMN) \
455 (DMN)->t_wakeup = NULL; \
456 thread_add_timer_msec(master, wakeup_down, (DMN), \
457 FUZZY(gs.period), &(DMN)->t_wakeup); \
460 #define SET_WAKEUP_UNRESPONSIVE(DMN) \
462 (DMN)->t_wakeup = NULL; \
463 thread_add_timer_msec(master, wakeup_unresponsive, (DMN), \
464 FUZZY(gs.period), &(DMN)->t_wakeup); \
467 #define SET_WAKEUP_ECHO(DMN) \
469 (DMN)->t_wakeup = NULL; \
470 thread_add_timer_msec(master, wakeup_send_echo, (DMN), \
471 FUZZY(gs.period), &(DMN)->t_wakeup); \
474 static int wakeup_down(struct thread
*t_wakeup
)
476 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
478 dmn
->t_wakeup
= NULL
;
479 if (try_connect(dmn
) < 0)
480 SET_WAKEUP_DOWN(dmn
);
481 if ((dmn
->connect_tries
> 1) && (dmn
->state
!= DAEMON_UP
))
486 static int wakeup_init(struct thread
*t_wakeup
)
488 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
490 dmn
->t_wakeup
= NULL
;
491 if (try_connect(dmn
) < 0) {
492 SET_WAKEUP_DOWN(dmn
);
493 flog_err(WATCHFRR_ERR_CONNECTION
,
494 "%s state -> down : initial connection attempt failed",
496 dmn
->state
= DAEMON_DOWN
;
501 static void daemon_down(struct daemon
*dmn
, const char *why
)
503 if (IS_UP(dmn
) || (dmn
->state
== DAEMON_INIT
))
504 flog_err(WATCHFRR_ERR_CONNECTION
,
505 "%s state -> down : %s", dmn
->name
, why
);
506 else if (gs
.loglevel
> LOG_DEBUG
)
507 zlog_debug("%s still down : %s", dmn
->name
, why
);
510 dmn
->state
= DAEMON_DOWN
;
515 THREAD_OFF(dmn
->t_read
);
516 THREAD_OFF(dmn
->t_write
);
517 THREAD_OFF(dmn
->t_wakeup
);
518 if (try_connect(dmn
) < 0)
519 SET_WAKEUP_DOWN(dmn
);
523 static int handle_read(struct thread
*t_read
)
525 struct daemon
*dmn
= THREAD_ARG(t_read
);
526 static const char resp
[sizeof(PING_TOKEN
) + 4] = PING_TOKEN
"\n";
527 char buf
[sizeof(resp
) + 100];
529 struct timeval delay
;
532 if ((rc
= read(dmn
->fd
, buf
, sizeof(buf
))) < 0) {
535 if (ERRNO_IO_RETRY(errno
)) {
536 /* Pretend it never happened. */
537 SET_READ_HANDLER(dmn
);
540 snprintf(why
, sizeof(why
), "unexpected read error: %s",
541 safe_strerror(errno
));
542 daemon_down(dmn
, why
);
546 daemon_down(dmn
, "read returned EOF");
549 if (!dmn
->echo_sent
.tv_sec
) {
550 char why
[sizeof(buf
) + 100];
551 snprintf(why
, sizeof(why
),
552 "unexpected read returns %d bytes: %.*s", (int)rc
,
554 daemon_down(dmn
, why
);
558 /* We are expecting an echo response: is there any chance that the
559 response would not be returned entirely in the first read? That
560 seems inconceivable... */
561 if ((rc
!= sizeof(resp
)) || memcmp(buf
, resp
, sizeof(resp
))) {
562 char why
[100 + sizeof(buf
)];
563 snprintf(why
, sizeof(why
),
564 "read returned bad echo response of %d bytes "
565 "(expecting %u): %.*s",
566 (int)rc
, (unsigned int)sizeof(resp
), (int)rc
, buf
);
567 daemon_down(dmn
, why
);
571 time_elapsed(&delay
, &dmn
->echo_sent
);
572 dmn
->echo_sent
.tv_sec
= 0;
573 if (dmn
->state
== DAEMON_UNRESPONSIVE
) {
574 if (delay
.tv_sec
< gs
.timeout
) {
575 dmn
->state
= DAEMON_UP
;
577 "%s state -> up : echo response received after %ld.%06ld "
579 dmn
->name
, (long)delay
.tv_sec
,
580 (long)delay
.tv_usec
);
583 "%s: slow echo response finally received after %ld.%06ld "
585 dmn
->name
, (long)delay
.tv_sec
,
586 (long)delay
.tv_usec
);
587 } else if (gs
.loglevel
> LOG_DEBUG
+ 1)
588 zlog_debug("%s: echo response received after %ld.%06ld seconds",
589 dmn
->name
, (long)delay
.tv_sec
, (long)delay
.tv_usec
);
591 SET_READ_HANDLER(dmn
);
593 thread_cancel(dmn
->t_wakeup
);
594 SET_WAKEUP_ECHO(dmn
);
600 * Wait till we notice that all daemons are ready before
601 * we send we are ready to systemd
603 static void daemon_send_ready(void)
606 if (!sent
&& gs
.numdown
== 0) {
609 fp
= fopen(DAEMON_VTY_DIR
"/watchfrr.started", "w");
612 #if defined HAVE_SYSTEMD
614 "Watchfrr: Notifying Systemd we are up and running");
615 systemd_send_started(master
, 0);
621 static void daemon_up(struct daemon
*dmn
, const char *why
)
623 dmn
->state
= DAEMON_UP
;
625 dmn
->connect_tries
= 0;
626 zlog_notice("%s state -> up : %s", dmn
->name
, why
);
628 SET_WAKEUP_ECHO(dmn
);
632 static int check_connect(struct thread
*t_write
)
634 struct daemon
*dmn
= THREAD_ARG(t_write
);
636 socklen_t reslen
= sizeof(sockerr
);
639 if (getsockopt(dmn
->fd
, SOL_SOCKET
, SO_ERROR
, (char *)&sockerr
, &reslen
)
641 zlog_warn("%s: check_connect: getsockopt failed: %s", dmn
->name
,
642 safe_strerror(errno
));
644 "getsockopt failed checking connection success");
647 if ((reslen
== sizeof(sockerr
)) && sockerr
) {
651 "getsockopt reports that connection attempt failed: %s",
652 safe_strerror(sockerr
));
653 daemon_down(dmn
, why
);
657 daemon_up(dmn
, "delayed connect succeeded");
661 static int wakeup_connect_hanging(struct thread
*t_wakeup
)
663 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
666 dmn
->t_wakeup
= NULL
;
667 snprintf(why
, sizeof(why
),
668 "connection attempt timed out after %ld seconds", gs
.timeout
);
669 daemon_down(dmn
, why
);
673 /* Making connection to protocol daemon. */
674 static int try_connect(struct daemon
*dmn
)
677 struct sockaddr_un addr
;
680 if (gs
.loglevel
> LOG_DEBUG
+ 1)
681 zlog_debug("%s: attempting to connect", dmn
->name
);
682 dmn
->connect_tries
++;
684 memset(&addr
, 0, sizeof(struct sockaddr_un
));
685 addr
.sun_family
= AF_UNIX
;
686 snprintf(addr
.sun_path
, sizeof(addr
.sun_path
), "%s/%s.vty", gs
.vtydir
,
688 #ifdef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN
689 len
= addr
.sun_len
= SUN_LEN(&addr
);
691 len
= sizeof(addr
.sun_family
) + strlen(addr
.sun_path
);
692 #endif /* HAVE_STRUCT_SOCKADDR_UN_SUN_LEN */
694 /* Quick check to see if we might succeed before we go to the trouble
695 of creating a socket. */
696 if (access(addr
.sun_path
, W_OK
) < 0) {
698 flog_err_sys(LIB_ERR_SYSTEM_CALL
,
699 "%s: access to socket %s denied: %s",
700 dmn
->name
, addr
.sun_path
,
701 safe_strerror(errno
));
705 if ((sock
= socket(AF_UNIX
, SOCK_STREAM
, 0)) < 0) {
706 flog_err_sys(LIB_ERR_SOCKET
, "%s(%s): cannot make socket: %s",
707 __func__
, addr
.sun_path
, safe_strerror(errno
));
711 if (set_nonblocking(sock
) < 0 || set_cloexec(sock
) < 0) {
712 flog_err_sys(LIB_ERR_SYSTEM_CALL
,
713 "%s(%s): set_nonblocking/cloexec(%d) failed",
714 __func__
, addr
.sun_path
, sock
);
719 if (connect(sock
, (struct sockaddr
*)&addr
, len
) < 0) {
720 if ((errno
!= EINPROGRESS
) && (errno
!= EWOULDBLOCK
)) {
721 if (gs
.loglevel
> LOG_DEBUG
)
722 zlog_debug("%s(%s): connect failed: %s",
723 __func__
, addr
.sun_path
,
724 safe_strerror(errno
));
728 if (gs
.loglevel
> LOG_DEBUG
)
729 zlog_debug("%s: connection in progress", dmn
->name
);
730 dmn
->state
= DAEMON_CONNECTING
;
733 thread_add_write(master
, check_connect
, dmn
, dmn
->fd
,
735 dmn
->t_wakeup
= NULL
;
736 thread_add_timer(master
, wakeup_connect_hanging
, dmn
,
737 gs
.timeout
, &dmn
->t_wakeup
);
738 SET_READ_HANDLER(dmn
);
743 SET_READ_HANDLER(dmn
);
744 daemon_up(dmn
, "connect succeeded");
748 static int phase_hanging(struct thread
*t_hanging
)
750 gs
.t_phase_hanging
= NULL
;
751 flog_err(WATCHFRR_ERR_CONNECTION
,
752 "Phase [%s] hanging for %ld seconds, aborting phased restart",
753 phase_str
[gs
.phase
], PHASE_TIMEOUT
);
754 gs
.phase
= PHASE_NONE
;
758 static void set_phase(restart_phase_t new_phase
)
760 gs
.phase
= new_phase
;
761 if (gs
.t_phase_hanging
)
762 thread_cancel(gs
.t_phase_hanging
);
763 gs
.t_phase_hanging
= NULL
;
764 thread_add_timer(master
, phase_hanging
, NULL
, PHASE_TIMEOUT
,
765 &gs
.t_phase_hanging
);
768 static void phase_check(void)
773 case PHASE_STOPS_PENDING
:
777 "Phased restart: all routing daemon stop jobs have completed.");
778 set_phase(PHASE_WAITING_DOWN
);
781 case PHASE_WAITING_DOWN
:
782 if (gs
.numdown
+ IS_UP(gs
.special
) < gs
.numdaemons
)
784 zlog_info("Phased restart: all routing daemons now down.");
785 run_job(&gs
.special
->restart
, "restart", gs
.restart_command
, 1,
787 set_phase(PHASE_ZEBRA_RESTART_PENDING
);
790 case PHASE_ZEBRA_RESTART_PENDING
:
791 if (gs
.special
->restart
.pid
)
793 zlog_info("Phased restart: %s restart job completed.",
795 set_phase(PHASE_WAITING_ZEBRA_UP
);
798 case PHASE_WAITING_ZEBRA_UP
:
799 if (!IS_UP(gs
.special
))
801 zlog_info("Phased restart: %s is now up.", gs
.special
->name
);
804 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
805 if (dmn
!= gs
.special
)
806 run_job(&dmn
->restart
, "start",
807 gs
.start_command
, 1, 0);
810 gs
.phase
= PHASE_NONE
;
811 THREAD_OFF(gs
.t_phase_hanging
);
812 zlog_notice("Phased global restart has completed.");
817 static void try_restart(struct daemon
*dmn
)
822 if (dmn
!= gs
.special
) {
823 if ((gs
.special
->state
== DAEMON_UP
)
824 && (gs
.phase
== PHASE_NONE
))
825 run_job(&dmn
->restart
, "restart", gs
.restart_command
, 0,
829 "%s: postponing restart attempt because master %s daemon "
830 "not up [%s], or phased restart in progress",
831 dmn
->name
, gs
.special
->name
,
832 state_str
[gs
.special
->state
]);
836 if ((gs
.phase
!= PHASE_NONE
) || gs
.numpids
) {
837 if (gs
.loglevel
> LOG_DEBUG
+ 1)
839 "postponing phased global restart: restart already in "
840 "progress [%s], or outstanding child processes [%d]",
841 phase_str
[gs
.phase
], gs
.numpids
);
844 /* Is it too soon for a restart? */
846 struct timeval delay
;
847 if (time_elapsed(&delay
, &gs
.special
->restart
.time
)->tv_sec
848 < gs
.special
->restart
.interval
) {
849 if (gs
.loglevel
> LOG_DEBUG
+ 1)
851 "postponing phased global restart: "
852 "elapsed time %ld < retry interval %ld",
854 gs
.special
->restart
.interval
);
858 run_job(&gs
.restart
, "restart", gs
.restart_command
, 0, 1);
861 static int wakeup_unresponsive(struct thread
*t_wakeup
)
863 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
865 dmn
->t_wakeup
= NULL
;
866 if (dmn
->state
!= DAEMON_UNRESPONSIVE
)
867 flog_err(WATCHFRR_ERR_CONNECTION
,
868 "%s: no longer unresponsive (now %s), "
869 "wakeup should have been cancelled!",
870 dmn
->name
, state_str
[dmn
->state
]);
872 SET_WAKEUP_UNRESPONSIVE(dmn
);
878 static int wakeup_no_answer(struct thread
*t_wakeup
)
880 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
882 dmn
->t_wakeup
= NULL
;
883 dmn
->state
= DAEMON_UNRESPONSIVE
;
884 flog_err(WATCHFRR_ERR_CONNECTION
,
885 "%s state -> unresponsive : no response yet to ping "
886 "sent %ld seconds ago",
887 dmn
->name
, gs
.timeout
);
888 SET_WAKEUP_UNRESPONSIVE(dmn
);
893 static int wakeup_send_echo(struct thread
*t_wakeup
)
895 static const char echocmd
[] = "echo " PING_TOKEN
;
897 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
899 dmn
->t_wakeup
= NULL
;
900 if (((rc
= write(dmn
->fd
, echocmd
, sizeof(echocmd
))) < 0)
901 || ((size_t)rc
!= sizeof(echocmd
))) {
902 char why
[100 + sizeof(echocmd
)];
903 snprintf(why
, sizeof(why
),
904 "write '%s' returned %d instead of %u", echocmd
,
905 (int)rc
, (unsigned int)sizeof(echocmd
));
906 daemon_down(dmn
, why
);
908 gettimeofday(&dmn
->echo_sent
, NULL
);
909 dmn
->t_wakeup
= NULL
;
910 thread_add_timer(master
, wakeup_no_answer
, dmn
, gs
.timeout
,
916 bool check_all_up(void)
920 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
921 if (dmn
->state
!= DAEMON_UP
)
926 static void sigint(void)
928 zlog_notice("Terminating on signal");
929 systemd_send_stopping();
933 static int valid_command(const char *cmd
)
937 return ((p
= strchr(cmd
, '%')) != NULL
) && (*(p
+ 1) == 's')
938 && !strchr(p
+ 1, '%');
941 /* This is an ugly hack to circumvent problems with passing command-line
942 arguments that contain spaces. The fix is to use a configuration file. */
943 static char *translate_blanks(const char *cmd
, const char *blankstr
)
947 size_t bslen
= strlen(blankstr
);
949 if (!(res
= strdup(cmd
))) {
953 while ((p
= strstr(res
, blankstr
)) != NULL
) {
956 memmove(p
+ 1, p
+ bslen
, strlen(p
+ bslen
) + 1);
961 struct zebra_privs_t watchfrr_privs
= {
963 .vty_group
= VTY_GROUP
,
967 static struct quagga_signal_t watchfrr_signals
[] = {
982 FRR_DAEMON_INFO(watchfrr
, WATCHFRR
,
983 .flags
= FRR_NO_PRIVSEP
| FRR_NO_TCPVTY
| FRR_LIMITED_CLI
984 | FRR_NO_CFG_PID_DRY
| FRR_NO_ZCLIENT
,
986 .printhelp
= printhelp
,
987 .copyright
= "Copyright 2004 Andrew J. Schorr",
989 .signals
= watchfrr_signals
,
990 .n_signals
= array_size(watchfrr_signals
),
992 .privs
= &watchfrr_privs
, )
994 #define DEPRECATED_OPTIONS "aAezR:"
996 int main(int argc
, char **argv
)
999 const char *pidfile
= pidfile_default
;
1000 const char *special
= "zebra";
1001 const char *blankstr
= NULL
;
1003 snprintf(pidfile_default
, sizeof(pidfile_default
), "%s/watchfrr.pid",
1006 frr_preinit(&watchfrr_di
, argc
, argv
);
1007 progname
= watchfrr_di
.progname
;
1009 frr_opt_add("b:dk:l:i:p:r:S:s:t:T:" DEPRECATED_OPTIONS
, longopts
, "");
1011 gs
.restart
.name
= "all";
1012 while ((opt
= frr_getopt(argc
, argv
, NULL
)) != EOF
) {
1013 if (opt
&& opt
< 128 && strchr(DEPRECATED_OPTIONS
, opt
)) {
1015 "The -%c option no longer exists.\n"
1016 "Please refer to the watchfrr(8) man page.\n",
1031 if (!valid_command(optarg
)) {
1033 "Invalid kill command, must contain '%%s': %s\n",
1037 gs
.stop_command
= optarg
;
1041 if ((sscanf(optarg
, "%d%1s", &gs
.loglevel
, garbage
)
1043 || (gs
.loglevel
< LOG_EMERG
)) {
1045 "Invalid loglevel argument: %s\n",
1050 case OPTION_MINRESTART
: {
1052 if ((sscanf(optarg
, "%ld%1s", &gs
.min_restart_interval
,
1055 || (gs
.min_restart_interval
< 0)) {
1057 "Invalid min_restart_interval argument: %s\n",
1062 case OPTION_MAXRESTART
: {
1064 if ((sscanf(optarg
, "%ld%1s", &gs
.max_restart_interval
,
1067 || (gs
.max_restart_interval
< 0)) {
1069 "Invalid max_restart_interval argument: %s\n",
1077 if ((sscanf(optarg
, "%d%1s", &period
, garbage
) != 1)
1078 || (gs
.period
< 1)) {
1080 "Invalid interval argument: %s\n",
1084 gs
.period
= 1000 * period
;
1090 if (!valid_command(optarg
)) {
1092 "Invalid restart command, must contain '%%s': %s\n",
1096 gs
.restart_command
= optarg
;
1099 if (!valid_command(optarg
)) {
1101 "Invalid start command, must contain '%%s': %s\n",
1105 gs
.start_command
= optarg
;
1112 if ((sscanf(optarg
, "%ld%1s", &gs
.timeout
, garbage
)
1114 || (gs
.timeout
< 1)) {
1116 "Invalid timeout argument: %s\n",
1123 if ((sscanf(optarg
, "%ld%1s", &gs
.restart_timeout
,
1126 || (gs
.restart_timeout
< 1)) {
1128 "Invalid restart timeout argument: %s\n",
1134 fputs("Invalid option.\n", stderr
);
1140 && (gs
.start_command
|| gs
.stop_command
|| gs
.restart_command
)) {
1141 fputs("Options -r/-s/-k are not used when --dry is active.\n",
1145 && (!gs
.restart_command
|| !gs
.start_command
|| !gs
.stop_command
)) {
1147 "Options -s (start), -k (kill), and -r (restart) are required.\n");
1152 if (gs
.restart_command
)
1153 gs
.restart_command
=
1154 translate_blanks(gs
.restart_command
, blankstr
);
1155 if (gs
.start_command
)
1157 translate_blanks(gs
.start_command
, blankstr
);
1158 if (gs
.stop_command
)
1160 translate_blanks(gs
.stop_command
, blankstr
);
1163 gs
.restart
.interval
= gs
.min_restart_interval
;
1165 master
= frr_init();
1166 watchfrr_error_init();
1168 zlog_set_level(ZLOG_DEST_MONITOR
, ZLOG_DISABLED
);
1169 if (watchfrr_di
.daemon_mode
) {
1170 zlog_set_level(ZLOG_DEST_SYSLOG
, MIN(gs
.loglevel
, LOG_DEBUG
));
1171 if (daemon(0, 0) < 0) {
1172 fprintf(stderr
, "Watchfrr daemon failed: %s",
1177 zlog_set_level(ZLOG_DEST_STDOUT
, MIN(gs
.loglevel
, LOG_DEBUG
));
1179 watchfrr_vty_init();
1185 struct daemon
*tail
= NULL
;
1187 for (i
= optind
; i
< argc
; i
++) {
1190 if (!(dmn
= (struct daemon
*)calloc(1, sizeof(*dmn
)))) {
1191 fprintf(stderr
, "calloc(1,%u) failed: %s\n",
1192 (unsigned int)sizeof(*dmn
),
1193 safe_strerror(errno
));
1196 dmn
->name
= dmn
->restart
.name
= argv
[i
];
1197 dmn
->state
= DAEMON_INIT
;
1201 dmn
->t_wakeup
= NULL
;
1202 thread_add_timer_msec(master
, wakeup_init
, dmn
,
1203 100 + (random() % 900),
1205 dmn
->restart
.interval
= gs
.min_restart_interval
;
1212 if (!strcmp(dmn
->name
, special
))
1217 fputs("Must specify one or more daemons to monitor.\n", stderr
);
1220 if (!watch_only
&& !gs
.special
) {
1221 fprintf(stderr
, "\"%s\" daemon must be in daemon list\n",
1226 /* Make sure we're not already running. */
1227 pid_output(pidfile
);
1229 /* Announce which daemons are being monitored. */
1234 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
1235 len
+= strlen(dmn
->name
) + 1;
1241 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
1244 strcpy(p
, dmn
->name
);
1247 zlog_notice("%s %s watching [%s]%s", progname
,
1249 watch_only
? ", monitor mode" : "");
1254 struct thread thread
;
1256 while (thread_fetch(master
, &thread
))
1257 thread_call(&thread
);
1260 systemd_send_stopping();