2 * Monitor status of frr daemons and restart if necessary.
4 * Copyright (C) 2004 Andrew J. Schorr
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; see the file COPYING; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26 #include <lib/version.h>
28 #include "memory_vty.h"
40 #define MIN(X,Y) (((X) <= (Y)) ? (X) : (Y))
43 /* Macros to help randomize timers. */
44 #define JITTER(X) ((random() % ((X)+1))-((X)/2))
45 #define FUZZY(X) ((X)+JITTER((X)/20))
47 #define DEFAULT_PERIOD 5
48 #define DEFAULT_TIMEOUT 10
49 #define DEFAULT_RESTART_TIMEOUT 20
50 #define DEFAULT_LOGLEVEL LOG_INFO
51 #define DEFAULT_MIN_RESTART 60
52 #define DEFAULT_MAX_RESTART 600
53 #ifdef PATH_WATCHFRR_PID
54 #define DEFAULT_PIDFILE PATH_WATCHFRR_PID
56 #define DEFAULT_PIDFILE STATEDIR "/watchfrr.pid"
59 #define VTYDIR DAEMON_VTY_DIR
61 #define VTYDIR STATEDIR
64 #define PING_TOKEN "PING"
66 /* Needs to be global, referenced somewhere inside libfrr. */
67 struct thread_master
*master
;
69 static bool watch_only
= false;
75 PHASE_ZEBRA_RESTART_PENDING
,
76 PHASE_WAITING_ZEBRA_UP
79 static const char *phase_str
[] = {
82 "Waiting for other daemons to come down",
83 "Zebra restart job running",
84 "Waiting for zebra to come up",
88 #define PHASE_TIMEOUT (3*gs.restart_timeout)
96 struct thread
*t_kill
;
100 static struct global_state
{
101 restart_phase_t phase
;
102 struct thread
*t_phase_hanging
;
106 long restart_timeout
;
107 long min_restart_interval
;
108 long max_restart_interval
;
109 struct daemon
*daemons
;
110 const char *restart_command
;
111 const char *start_command
;
112 const char *stop_command
;
113 struct restart_info restart
;
114 int unresponsive_restart
;
116 struct daemon
*special
; /* points to zebra when doing phased restart */
119 int numdown
; /* # of daemons that are not UP or UNRESPONSIVE */
123 .period
= 1000 * DEFAULT_PERIOD
,
124 .timeout
= DEFAULT_TIMEOUT
,
125 .restart_timeout
= DEFAULT_RESTART_TIMEOUT
,
126 .loglevel
= DEFAULT_LOGLEVEL
,
127 .min_restart_interval
= DEFAULT_MIN_RESTART
,
128 .max_restart_interval
= DEFAULT_MAX_RESTART
,
140 (((DMN)->state == DAEMON_UP) || ((DMN)->state == DAEMON_UNRESPONSIVE))
142 static const char *state_str
[] = {
143 "Init", "Down", "Connecting", "Up", "Unresponsive",
148 daemon_state_t state
;
150 struct timeval echo_sent
;
152 struct thread
*t_wakeup
;
153 struct thread
*t_read
;
154 struct thread
*t_write
;
156 struct restart_info restart
;
159 #define OPTION_MINRESTART 2000
160 #define OPTION_MAXRESTART 2001
161 #define OPTION_DRY 2002
163 static const struct option longopts
[] = {
164 {"daemon", no_argument
, NULL
, 'd'},
165 {"statedir", required_argument
, NULL
, 'S'},
166 {"loglevel", required_argument
, NULL
, 'l'},
167 {"interval", required_argument
, NULL
, 'i'},
168 {"timeout", required_argument
, NULL
, 't'},
169 {"restart-timeout", required_argument
, NULL
, 'T'},
170 {"restart", required_argument
, NULL
, 'r'},
171 {"start-command", required_argument
, NULL
, 's'},
172 {"kill-command", required_argument
, NULL
, 'k'},
173 {"unresponsive-restart", no_argument
, NULL
, 'z'},
174 {"dry", no_argument
, NULL
, OPTION_DRY
},
175 {"min-restart-interval", required_argument
, NULL
, OPTION_MINRESTART
},
176 {"max-restart-interval", required_argument
, NULL
, OPTION_MAXRESTART
},
177 {"pid-file", required_argument
, NULL
, 'p'},
178 {"blank-string", required_argument
, NULL
, 'b'},
179 {"help", no_argument
, NULL
, 'h'},
180 {"version", no_argument
, NULL
, 'v'},
183 static int try_connect(struct daemon
*dmn
);
184 static int wakeup_send_echo(struct thread
*t_wakeup
);
185 static void try_restart(struct daemon
*dmn
);
186 static void phase_check(void);
188 static const char *progname
;
189 static void printhelp(FILE *target
)
192 "Usage : %s [OPTION...] <daemon name> ...\n\n\
193 Watchdog program to monitor status of frr daemons and try to restart\n\
194 them if they are down or unresponsive. It determines whether a daemon is\n\
195 up based on whether it can connect to the daemon's vty unix stream socket.\n\
196 It then repeatedly sends echo commands over that socket to determine whether\n\
197 the daemon is responsive. If the daemon crashes, we will receive an EOF\n\
198 on the socket connection and know immediately that the daemon is down.\n\n\
199 The daemons to be monitored should be listed on the command line.\n\n\
200 In order to avoid attempting to restart the daemons in a fast loop,\n\
201 the -m and -M options allow you to control the minimum delay between\n\
202 restart commands. The minimum restart delay is recalculated each time\n\
203 a restart is attempted: if the time since the last restart attempt exceeds\n\
204 twice the -M value, then the restart delay is set to the -m value.\n\
205 Otherwise, the interval is doubled (but capped at the -M value).\n\n",
210 -d, --daemon Run in daemon mode. In this mode, error messages are sent\n\
211 to syslog instead of stdout.\n\
212 -S, --statedir Set the vty socket directory (default is %s)\n\
213 -l, --loglevel Set the logging level (default is %d).\n\
214 The value should range from %d (LOG_EMERG) to %d (LOG_DEBUG),\n\
215 but it can be set higher than %d if extra-verbose debugging\n\
216 messages are desired.\n\
217 --min-restart-interval\n\
218 Set the minimum seconds to wait between invocations of daemon\n\
219 restart commands (default is %d).\n\
220 --max-restart-interval\n\
221 Set the maximum seconds to wait between invocations of daemon\n\
222 restart commands (default is %d).\n\
223 -i, --interval Set the status polling interval in seconds (default is %d)\n\
224 -t, --timeout Set the unresponsiveness timeout in seconds (default is %d)\n\
225 -T, --restart-timeout\n\
226 Set the restart (kill) timeout in seconds (default is %d).\n\
227 If any background jobs are still running after this much\n\
228 time has elapsed, they will be killed.\n\
229 -r, --restart Supply a Bourne shell command to use to restart a single\n\
230 daemon. The command string should include '%%s' where the\n\
231 name of the daemon should be substituted.\n\
232 -s, --start-command\n\
233 Supply a Bourne shell to command to use to start a single\n\
234 daemon. The command string should include '%%s' where the\n\
235 name of the daemon should be substituted.\n\
236 -k, --kill-command\n\
237 Supply a Bourne shell to command to use to stop a single\n\
238 daemon. The command string should include '%%s' where the\n\
239 name of the daemon should be substituted.\n\
240 -z, --unresponsive-restart\n\
241 When a daemon is unresponsive, treat it as being down for\n\
243 --dry Do not start or restart anything, just log.\n\
244 -p, --pid-file Set process identifier file name\n\
246 -b, --blank-string\n\
247 When the supplied argument string is found in any of the\n\
248 various shell command arguments (-r, -s, or -k), replace\n\
249 it with a space. This is an ugly hack to circumvent problems\n\
250 passing command-line arguments with embedded spaces.\n\
251 -v, --version Print program version\n\
252 -h, --help Display this help and exit\n",
253 VTYDIR
, DEFAULT_LOGLEVEL
, LOG_EMERG
, LOG_DEBUG
, LOG_DEBUG
,
254 DEFAULT_MIN_RESTART
, DEFAULT_MAX_RESTART
, DEFAULT_PERIOD
,
255 DEFAULT_TIMEOUT
, DEFAULT_RESTART_TIMEOUT
, DEFAULT_PIDFILE
);
258 static pid_t
run_background(char *shell_cmd
)
262 switch (child
= fork()) {
264 zlog_err("fork failed, cannot run command [%s]: %s", shell_cmd
,
265 safe_strerror(errno
));
269 /* Use separate process group so child processes can be killed
271 if (setpgid(0, 0) < 0)
272 zlog_warn("warning: setpgid(0,0) failed: %s",
273 safe_strerror(errno
));
277 char *const argv
[4] = {shell
, dashc
, shell_cmd
, NULL
};
278 execv("/bin/sh", argv
);
279 zlog_err("execv(/bin/sh -c '%s') failed: %s", shell_cmd
,
280 safe_strerror(errno
));
284 /* Parent process: we will reap the child later. */
285 zlog_err("Forked background command [pid %d]: %s", (int)child
,
291 static struct timeval
*time_elapsed(struct timeval
*result
,
292 const struct timeval
*start_time
)
294 gettimeofday(result
, NULL
);
295 result
->tv_sec
-= start_time
->tv_sec
;
296 result
->tv_usec
-= start_time
->tv_usec
;
297 while (result
->tv_usec
< 0) {
298 result
->tv_usec
+= 1000000L;
304 static int restart_kill(struct thread
*t_kill
)
306 struct restart_info
*restart
= THREAD_ARG(t_kill
);
307 struct timeval delay
;
309 time_elapsed(&delay
, &restart
->time
);
311 "Warning: %s %s child process %d still running after "
312 "%ld seconds, sending signal %d",
313 restart
->what
, restart
->name
, (int)restart
->pid
,
314 (long)delay
.tv_sec
, (restart
->kills
? SIGKILL
: SIGTERM
));
315 kill(-restart
->pid
, (restart
->kills
? SIGKILL
: SIGTERM
));
317 restart
->t_kill
= NULL
;
318 thread_add_timer(master
, restart_kill
, restart
, gs
.restart_timeout
,
323 static struct restart_info
*find_child(pid_t child
)
326 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
327 if (dmn
->restart
.pid
== child
)
328 return &dmn
->restart
;
333 static void sigchild(void)
339 struct restart_info
*restart
;
341 switch (child
= waitpid(-1, &status
, WNOHANG
)) {
343 zlog_err("waitpid failed: %s", safe_strerror(errno
));
346 zlog_warn("SIGCHLD received, but waitpid did not reap a child");
350 if (child
== integrated_write_pid
) {
351 integrated_write_sigchld(status
);
355 if ((restart
= find_child(child
)) != NULL
) {
356 name
= restart
->name
;
357 what
= restart
->what
;
360 thread_cancel(restart
->t_kill
);
361 restart
->t_kill
= NULL
;
362 /* Update restart time to reflect the time the command
364 gettimeofday(&restart
->time
, NULL
);
367 "waitpid returned status for an unknown child process %d",
372 if (WIFSTOPPED(status
))
373 zlog_warn("warning: %s %s process %d is stopped", what
, name
,
375 else if (WIFSIGNALED(status
))
376 zlog_warn("%s %s process %d terminated due to signal %d", what
,
377 name
, (int)child
, WTERMSIG(status
));
378 else if (WIFEXITED(status
)) {
379 if (WEXITSTATUS(status
) != 0)
381 "%s %s process %d exited with non-zero status %d",
382 what
, name
, (int)child
, WEXITSTATUS(status
));
384 zlog_debug("%s %s process %d exited normally", what
,
387 zlog_err("cannot interpret %s %s process %d wait status 0x%x",
388 what
, name
, (int)child
, status
);
392 static int run_job(struct restart_info
*restart
, const char *cmdtype
,
393 const char *command
, int force
, int update_interval
)
395 struct timeval delay
;
397 if (gs
.loglevel
> LOG_DEBUG
+ 1)
398 zlog_debug("attempting to %s %s", cmdtype
, restart
->name
);
401 if (gs
.loglevel
> LOG_DEBUG
+ 1)
403 "cannot %s %s, previous pid %d still running",
404 cmdtype
, restart
->name
, (int)restart
->pid
);
408 /* Note: time_elapsed test must come before the force test, since we
410 to make sure that delay is initialized for use below in updating the
412 if ((time_elapsed(&delay
, &restart
->time
)->tv_sec
< restart
->interval
)
414 if (gs
.loglevel
> LOG_DEBUG
+ 1)
417 "elapsed time %ld < retry interval %ld",
418 cmdtype
, restart
->name
, (long)delay
.tv_sec
,
423 gettimeofday(&restart
->time
, NULL
);
426 char cmd
[strlen(command
) + strlen(restart
->name
) + 1];
427 snprintf(cmd
, sizeof(cmd
), command
, restart
->name
);
428 if ((restart
->pid
= run_background(cmd
)) > 0) {
429 restart
->t_kill
= NULL
;
430 thread_add_timer(master
, restart_kill
, restart
,
431 gs
.restart_timeout
, &restart
->t_kill
);
432 restart
->what
= cmdtype
;
438 /* Calculate the new restart interval. */
439 if (update_interval
) {
440 if (delay
.tv_sec
> 2 * gs
.max_restart_interval
)
441 restart
->interval
= gs
.min_restart_interval
;
442 else if ((restart
->interval
*= 2) > gs
.max_restart_interval
)
443 restart
->interval
= gs
.max_restart_interval
;
444 if (gs
.loglevel
> LOG_DEBUG
+ 1)
445 zlog_debug("restart %s interval is now %ld",
446 restart
->name
, restart
->interval
);
451 #define SET_READ_HANDLER(DMN) \
453 (DMN)->t_read = NULL; \
454 thread_add_read(master, handle_read, (DMN), (DMN)->fd, \
458 #define SET_WAKEUP_DOWN(DMN) \
460 (DMN)->t_wakeup = NULL; \
461 thread_add_timer_msec(master, wakeup_down, (DMN), \
462 FUZZY(gs.period), &(DMN)->t_wakeup); \
465 #define SET_WAKEUP_UNRESPONSIVE(DMN) \
467 (DMN)->t_wakeup = NULL; \
468 thread_add_timer_msec(master, wakeup_unresponsive, (DMN), \
469 FUZZY(gs.period), &(DMN)->t_wakeup); \
472 #define SET_WAKEUP_ECHO(DMN) \
474 (DMN)->t_wakeup = NULL; \
475 thread_add_timer_msec(master, wakeup_send_echo, (DMN), \
476 FUZZY(gs.period), &(DMN)->t_wakeup); \
479 static int wakeup_down(struct thread
*t_wakeup
)
481 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
483 dmn
->t_wakeup
= NULL
;
484 if (try_connect(dmn
) < 0)
485 SET_WAKEUP_DOWN(dmn
);
486 if ((dmn
->connect_tries
> 1) && (dmn
->state
!= DAEMON_UP
))
491 static int wakeup_init(struct thread
*t_wakeup
)
493 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
495 dmn
->t_wakeup
= NULL
;
496 if (try_connect(dmn
) < 0) {
497 SET_WAKEUP_DOWN(dmn
);
498 zlog_err("%s state -> down : initial connection attempt failed",
500 dmn
->state
= DAEMON_DOWN
;
505 static void daemon_down(struct daemon
*dmn
, const char *why
)
507 if (IS_UP(dmn
) || (dmn
->state
== DAEMON_INIT
))
508 zlog_err("%s state -> down : %s", dmn
->name
, why
);
509 else if (gs
.loglevel
> LOG_DEBUG
)
510 zlog_debug("%s still down : %s", dmn
->name
, why
);
513 dmn
->state
= DAEMON_DOWN
;
518 THREAD_OFF(dmn
->t_read
);
519 THREAD_OFF(dmn
->t_write
);
520 THREAD_OFF(dmn
->t_wakeup
);
521 if (try_connect(dmn
) < 0)
522 SET_WAKEUP_DOWN(dmn
);
526 static int handle_read(struct thread
*t_read
)
528 struct daemon
*dmn
= THREAD_ARG(t_read
);
529 static const char resp
[sizeof(PING_TOKEN
) + 4] = PING_TOKEN
"\n";
530 char buf
[sizeof(resp
) + 100];
532 struct timeval delay
;
535 if ((rc
= read(dmn
->fd
, buf
, sizeof(buf
))) < 0) {
538 if (ERRNO_IO_RETRY(errno
)) {
539 /* Pretend it never happened. */
540 SET_READ_HANDLER(dmn
);
543 snprintf(why
, sizeof(why
), "unexpected read error: %s",
544 safe_strerror(errno
));
545 daemon_down(dmn
, why
);
549 daemon_down(dmn
, "read returned EOF");
552 if (!dmn
->echo_sent
.tv_sec
) {
553 char why
[sizeof(buf
) + 100];
554 snprintf(why
, sizeof(why
),
555 "unexpected read returns %d bytes: %.*s", (int)rc
,
557 daemon_down(dmn
, why
);
561 /* We are expecting an echo response: is there any chance that the
562 response would not be returned entirely in the first read? That
563 seems inconceivable... */
564 if ((rc
!= sizeof(resp
)) || memcmp(buf
, resp
, sizeof(resp
))) {
565 char why
[100 + sizeof(buf
)];
566 snprintf(why
, sizeof(why
),
567 "read returned bad echo response of %d bytes "
568 "(expecting %u): %.*s",
569 (int)rc
, (u_int
)sizeof(resp
), (int)rc
, buf
);
570 daemon_down(dmn
, why
);
574 time_elapsed(&delay
, &dmn
->echo_sent
);
575 dmn
->echo_sent
.tv_sec
= 0;
576 if (dmn
->state
== DAEMON_UNRESPONSIVE
) {
577 if (delay
.tv_sec
< gs
.timeout
) {
578 dmn
->state
= DAEMON_UP
;
580 "%s state -> up : echo response received after %ld.%06ld "
582 dmn
->name
, (long)delay
.tv_sec
,
583 (long)delay
.tv_usec
);
586 "%s: slow echo response finally received after %ld.%06ld "
588 dmn
->name
, (long)delay
.tv_sec
,
589 (long)delay
.tv_usec
);
590 } else if (gs
.loglevel
> LOG_DEBUG
+ 1)
591 zlog_debug("%s: echo response received after %ld.%06ld seconds",
592 dmn
->name
, (long)delay
.tv_sec
, (long)delay
.tv_usec
);
594 SET_READ_HANDLER(dmn
);
596 thread_cancel(dmn
->t_wakeup
);
597 SET_WAKEUP_ECHO(dmn
);
603 * Wait till we notice that all daemons are ready before
604 * we send we are ready to systemd
606 static void daemon_send_ready(void)
609 if (!sent
&& gs
.numdown
== 0) {
612 fp
= fopen(DAEMON_VTY_DIR
"/watchfrr.started", "w");
615 "Watchfrr: Notifying Systemd we are up and running");
616 systemd_send_started(master
, 0);
621 static void daemon_up(struct daemon
*dmn
, const char *why
)
623 dmn
->state
= DAEMON_UP
;
625 dmn
->connect_tries
= 0;
626 zlog_notice("%s state -> up : %s", dmn
->name
, why
);
628 SET_WAKEUP_ECHO(dmn
);
632 static int check_connect(struct thread
*t_write
)
634 struct daemon
*dmn
= THREAD_ARG(t_write
);
636 socklen_t reslen
= sizeof(sockerr
);
639 if (getsockopt(dmn
->fd
, SOL_SOCKET
, SO_ERROR
, (char *)&sockerr
, &reslen
)
641 zlog_warn("%s: check_connect: getsockopt failed: %s", dmn
->name
,
642 safe_strerror(errno
));
644 "getsockopt failed checking connection success");
647 if ((reslen
== sizeof(sockerr
)) && sockerr
) {
651 "getsockopt reports that connection attempt failed: %s",
652 safe_strerror(sockerr
));
653 daemon_down(dmn
, why
);
657 daemon_up(dmn
, "delayed connect succeeded");
661 static int wakeup_connect_hanging(struct thread
*t_wakeup
)
663 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
666 dmn
->t_wakeup
= NULL
;
667 snprintf(why
, sizeof(why
),
668 "connection attempt timed out after %ld seconds", gs
.timeout
);
669 daemon_down(dmn
, why
);
673 /* Making connection to protocol daemon. */
674 static int try_connect(struct daemon
*dmn
)
677 struct sockaddr_un addr
;
680 if (gs
.loglevel
> LOG_DEBUG
+ 1)
681 zlog_debug("%s: attempting to connect", dmn
->name
);
682 dmn
->connect_tries
++;
684 memset(&addr
, 0, sizeof(struct sockaddr_un
));
685 addr
.sun_family
= AF_UNIX
;
686 snprintf(addr
.sun_path
, sizeof(addr
.sun_path
), "%s/%s.vty", gs
.vtydir
,
688 #ifdef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN
689 len
= addr
.sun_len
= SUN_LEN(&addr
);
691 len
= sizeof(addr
.sun_family
) + strlen(addr
.sun_path
);
692 #endif /* HAVE_STRUCT_SOCKADDR_UN_SUN_LEN */
694 /* Quick check to see if we might succeed before we go to the trouble
695 of creating a socket. */
696 if (access(addr
.sun_path
, W_OK
) < 0) {
698 zlog_err("%s: access to socket %s denied: %s",
699 dmn
->name
, addr
.sun_path
,
700 safe_strerror(errno
));
704 if ((sock
= socket(AF_UNIX
, SOCK_STREAM
, 0)) < 0) {
705 zlog_err("%s(%s): cannot make socket: %s", __func__
,
706 addr
.sun_path
, safe_strerror(errno
));
710 if (set_nonblocking(sock
) < 0 || set_cloexec(sock
) < 0) {
711 zlog_err("%s(%s): set_nonblocking/cloexec(%d) failed", __func__
,
712 addr
.sun_path
, sock
);
717 if (connect(sock
, (struct sockaddr
*)&addr
, len
) < 0) {
718 if ((errno
!= EINPROGRESS
) && (errno
!= EWOULDBLOCK
)) {
719 if (gs
.loglevel
> LOG_DEBUG
)
720 zlog_debug("%s(%s): connect failed: %s",
721 __func__
, addr
.sun_path
,
722 safe_strerror(errno
));
726 if (gs
.loglevel
> LOG_DEBUG
)
727 zlog_debug("%s: connection in progress", dmn
->name
);
728 dmn
->state
= DAEMON_CONNECTING
;
731 thread_add_write(master
, check_connect
, dmn
, dmn
->fd
,
733 dmn
->t_wakeup
= NULL
;
734 thread_add_timer(master
, wakeup_connect_hanging
, dmn
,
735 gs
.timeout
, &dmn
->t_wakeup
);
736 SET_READ_HANDLER(dmn
);
741 SET_READ_HANDLER(dmn
);
742 daemon_up(dmn
, "connect succeeded");
746 static int phase_hanging(struct thread
*t_hanging
)
748 gs
.t_phase_hanging
= NULL
;
749 zlog_err("Phase [%s] hanging for %ld seconds, aborting phased restart",
750 phase_str
[gs
.phase
], PHASE_TIMEOUT
);
751 gs
.phase
= PHASE_NONE
;
755 static void set_phase(restart_phase_t new_phase
)
757 gs
.phase
= new_phase
;
758 if (gs
.t_phase_hanging
)
759 thread_cancel(gs
.t_phase_hanging
);
760 gs
.t_phase_hanging
= NULL
;
761 thread_add_timer(master
, phase_hanging
, NULL
, PHASE_TIMEOUT
,
762 &gs
.t_phase_hanging
);
765 static void phase_check(void)
770 case PHASE_STOPS_PENDING
:
774 "Phased restart: all routing daemon stop jobs have completed.");
775 set_phase(PHASE_WAITING_DOWN
);
778 case PHASE_WAITING_DOWN
:
779 if (gs
.numdown
+ IS_UP(gs
.special
) < gs
.numdaemons
)
781 zlog_info("Phased restart: all routing daemons now down.");
782 run_job(&gs
.special
->restart
, "restart", gs
.restart_command
, 1,
784 set_phase(PHASE_ZEBRA_RESTART_PENDING
);
787 case PHASE_ZEBRA_RESTART_PENDING
:
788 if (gs
.special
->restart
.pid
)
790 zlog_info("Phased restart: %s restart job completed.",
792 set_phase(PHASE_WAITING_ZEBRA_UP
);
795 case PHASE_WAITING_ZEBRA_UP
:
796 if (!IS_UP(gs
.special
))
798 zlog_info("Phased restart: %s is now up.", gs
.special
->name
);
801 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
802 if (dmn
!= gs
.special
)
803 run_job(&dmn
->restart
, "start",
804 gs
.start_command
, 1, 0);
807 gs
.phase
= PHASE_NONE
;
808 THREAD_OFF(gs
.t_phase_hanging
);
809 zlog_notice("Phased global restart has completed.");
814 static void try_restart(struct daemon
*dmn
)
819 if (dmn
!= gs
.special
) {
820 if ((gs
.special
->state
== DAEMON_UP
)
821 && (gs
.phase
== PHASE_NONE
))
822 run_job(&dmn
->restart
, "restart", gs
.restart_command
, 0,
826 "%s: postponing restart attempt because master %s daemon "
827 "not up [%s], or phased restart in progress",
828 dmn
->name
, gs
.special
->name
,
829 state_str
[gs
.special
->state
]);
833 if ((gs
.phase
!= PHASE_NONE
) || gs
.numpids
) {
834 if (gs
.loglevel
> LOG_DEBUG
+ 1)
836 "postponing phased global restart: restart already in "
837 "progress [%s], or outstanding child processes [%d]",
838 phase_str
[gs
.phase
], gs
.numpids
);
841 /* Is it too soon for a restart? */
843 struct timeval delay
;
844 if (time_elapsed(&delay
, &gs
.special
->restart
.time
)->tv_sec
845 < gs
.special
->restart
.interval
) {
846 if (gs
.loglevel
> LOG_DEBUG
+ 1)
848 "postponing phased global restart: "
849 "elapsed time %ld < retry interval %ld",
851 gs
.special
->restart
.interval
);
855 run_job(&gs
.restart
, "restart", gs
.restart_command
, 0, 1);
858 static int wakeup_unresponsive(struct thread
*t_wakeup
)
860 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
862 dmn
->t_wakeup
= NULL
;
863 if (dmn
->state
!= DAEMON_UNRESPONSIVE
)
865 "%s: no longer unresponsive (now %s), "
866 "wakeup should have been cancelled!",
867 dmn
->name
, state_str
[dmn
->state
]);
869 SET_WAKEUP_UNRESPONSIVE(dmn
);
875 static int wakeup_no_answer(struct thread
*t_wakeup
)
877 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
879 dmn
->t_wakeup
= NULL
;
880 dmn
->state
= DAEMON_UNRESPONSIVE
;
882 "%s state -> unresponsive : no response yet to ping "
883 "sent %ld seconds ago",
884 dmn
->name
, gs
.timeout
);
885 if (gs
.unresponsive_restart
) {
886 SET_WAKEUP_UNRESPONSIVE(dmn
);
892 static int wakeup_send_echo(struct thread
*t_wakeup
)
894 static const char echocmd
[] = "echo " PING_TOKEN
;
896 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
898 dmn
->t_wakeup
= NULL
;
899 if (((rc
= write(dmn
->fd
, echocmd
, sizeof(echocmd
))) < 0)
900 || ((size_t)rc
!= sizeof(echocmd
))) {
901 char why
[100 + sizeof(echocmd
)];
902 snprintf(why
, sizeof(why
),
903 "write '%s' returned %d instead of %u", echocmd
,
904 (int)rc
, (u_int
)sizeof(echocmd
));
905 daemon_down(dmn
, why
);
907 gettimeofday(&dmn
->echo_sent
, NULL
);
908 dmn
->t_wakeup
= NULL
;
909 thread_add_timer(master
, wakeup_no_answer
, dmn
, gs
.timeout
,
915 static void sigint(void)
917 zlog_notice("Terminating on signal");
918 systemd_send_stopping();
922 static int valid_command(const char *cmd
)
926 return ((p
= strchr(cmd
, '%')) != NULL
) && (*(p
+ 1) == 's')
927 && !strchr(p
+ 1, '%');
930 /* This is an ugly hack to circumvent problems with passing command-line
931 arguments that contain spaces. The fix is to use a configuration file. */
932 static char *translate_blanks(const char *cmd
, const char *blankstr
)
936 size_t bslen
= strlen(blankstr
);
938 if (!(res
= strdup(cmd
))) {
942 while ((p
= strstr(res
, blankstr
)) != NULL
) {
945 memmove(p
+ 1, p
+ bslen
, strlen(p
+ bslen
) + 1);
950 struct zebra_privs_t watchfrr_privs
= {
952 .vty_group
= VTY_GROUP
,
956 static struct quagga_signal_t watchfrr_signals
[] = {
971 FRR_DAEMON_INFO(watchfrr
, WATCHFRR
,
972 .flags
= FRR_NO_PRIVSEP
| FRR_NO_TCPVTY
| FRR_LIMITED_CLI
973 | FRR_NO_CFG_PID_DRY
| FRR_NO_ZCLIENT
,
975 .printhelp
= printhelp
,
976 .copyright
= "Copyright 2004 Andrew J. Schorr",
978 .signals
= watchfrr_signals
,
979 .n_signals
= array_size(watchfrr_signals
),
981 .privs
= &watchfrr_privs
, )
983 int main(int argc
, char **argv
)
986 const char *pidfile
= DEFAULT_PIDFILE
;
987 const char *special
= "zebra";
988 const char *blankstr
= NULL
;
990 frr_preinit(&watchfrr_di
, argc
, argv
);
991 progname
= watchfrr_di
.progname
;
993 frr_opt_add("b:dk:l:i:p:r:S:s:t:T:z", longopts
, "");
995 gs
.restart
.name
= "all";
996 while ((opt
= frr_getopt(argc
, argv
, NULL
)) != EOF
) {
1007 if (!valid_command(optarg
)) {
1009 "Invalid kill command, must contain '%%s': %s\n",
1013 gs
.stop_command
= optarg
;
1017 if ((sscanf(optarg
, "%d%1s", &gs
.loglevel
, garbage
)
1019 || (gs
.loglevel
< LOG_EMERG
)) {
1021 "Invalid loglevel argument: %s\n",
1026 case OPTION_MINRESTART
: {
1028 if ((sscanf(optarg
, "%ld%1s", &gs
.min_restart_interval
,
1031 || (gs
.min_restart_interval
< 0)) {
1033 "Invalid min_restart_interval argument: %s\n",
1038 case OPTION_MAXRESTART
: {
1040 if ((sscanf(optarg
, "%ld%1s", &gs
.max_restart_interval
,
1043 || (gs
.max_restart_interval
< 0)) {
1045 "Invalid max_restart_interval argument: %s\n",
1053 if ((sscanf(optarg
, "%d%1s", &period
, garbage
) != 1)
1054 || (gs
.period
< 1)) {
1056 "Invalid interval argument: %s\n",
1060 gs
.period
= 1000 * period
;
1066 if (!valid_command(optarg
)) {
1068 "Invalid restart command, must contain '%%s': %s\n",
1072 gs
.restart_command
= optarg
;
1075 if (!valid_command(optarg
)) {
1077 "Invalid start command, must contain '%%s': %s\n",
1081 gs
.start_command
= optarg
;
1088 if ((sscanf(optarg
, "%ld%1s", &gs
.timeout
, garbage
)
1090 || (gs
.timeout
< 1)) {
1092 "Invalid timeout argument: %s\n",
1099 if ((sscanf(optarg
, "%ld%1s", &gs
.restart_timeout
,
1102 || (gs
.restart_timeout
< 1)) {
1104 "Invalid restart timeout argument: %s\n",
1110 gs
.unresponsive_restart
= 1;
1113 fputs("Invalid option.\n", stderr
);
1118 if (watch_only
&& (gs
.unresponsive_restart
|| gs
.start_command
1119 || gs
.stop_command
|| gs
.restart_command
)) {
1120 fputs("Options -z/-r/-s/-k make no sense combined with -D.\n",
1125 && (!gs
.restart_command
|| !gs
.start_command
|| !gs
.stop_command
)) {
1127 "Options -s (start), -k (kill), and -r (restart) are required.\n");
1132 if (gs
.restart_command
)
1133 gs
.restart_command
=
1134 translate_blanks(gs
.restart_command
, blankstr
);
1135 if (gs
.start_command
)
1137 translate_blanks(gs
.start_command
, blankstr
);
1138 if (gs
.stop_command
)
1140 translate_blanks(gs
.stop_command
, blankstr
);
1143 gs
.restart
.interval
= gs
.min_restart_interval
;
1145 master
= frr_init();
1147 zlog_set_level(ZLOG_DEST_MONITOR
, ZLOG_DISABLED
);
1148 if (watchfrr_di
.daemon_mode
) {
1149 zlog_set_level(ZLOG_DEST_SYSLOG
, MIN(gs
.loglevel
, LOG_DEBUG
));
1150 if (daemon(0, 0) < 0) {
1151 fprintf(stderr
, "Watchfrr daemon failed: %s",
1156 zlog_set_level(ZLOG_DEST_STDOUT
, MIN(gs
.loglevel
, LOG_DEBUG
));
1158 watchfrr_vty_init();
1164 struct daemon
*tail
= NULL
;
1166 for (i
= optind
; i
< argc
; i
++) {
1169 if (!(dmn
= (struct daemon
*)calloc(1, sizeof(*dmn
)))) {
1170 fprintf(stderr
, "calloc(1,%u) failed: %s\n",
1171 (u_int
)sizeof(*dmn
),
1172 safe_strerror(errno
));
1175 dmn
->name
= dmn
->restart
.name
= argv
[i
];
1176 dmn
->state
= DAEMON_INIT
;
1180 dmn
->t_wakeup
= NULL
;
1181 thread_add_timer_msec(master
, wakeup_init
, dmn
,
1182 100 + (random() % 900),
1184 dmn
->restart
.interval
= gs
.min_restart_interval
;
1191 if (!strcmp(dmn
->name
, special
))
1196 fputs("Must specify one or more daemons to monitor.\n", stderr
);
1199 if (!watch_only
&& !gs
.special
) {
1200 fprintf(stderr
, "\"%s\" daemon must be in daemon list\n",
1205 /* Make sure we're not already running. */
1206 pid_output(pidfile
);
1208 /* Announce which daemons are being monitored. */
1213 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
1214 len
+= strlen(dmn
->name
) + 1;
1220 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
1223 strcpy(p
, dmn
->name
);
1226 zlog_notice("%s %s watching [%s]%s", progname
,
1228 watch_only
? ", monitor mode" : "");
1233 struct thread thread
;
1235 while (thread_fetch(master
, &thread
))
1236 thread_call(&thread
);
1239 systemd_send_stopping();