2 * Monitor status of frr daemons and restart if necessary.
4 * Copyright (C) 2004 Andrew J. Schorr
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; see the file COPYING; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26 #include <lib/version.h>
29 #include "lib_errors.h"
30 #include "zlog_targets.h"
41 #include "watchfrr_errors.h"
44 #define MIN(X,Y) (((X) <= (Y)) ? (X) : (Y))
47 /* Macros to help randomize timers. */
48 #define JITTER(X) ((frr_weak_random() % ((X)+1))-((X)/2))
49 #define FUZZY(X) ((X)+JITTER((X)/20))
51 #define DEFAULT_PERIOD 5
52 #define DEFAULT_TIMEOUT 90
53 #define DEFAULT_RESTART_TIMEOUT 20
54 #define DEFAULT_LOGLEVEL LOG_INFO
55 #define DEFAULT_MIN_RESTART 60
56 #define DEFAULT_MAX_RESTART 600
57 #define DEFAULT_OPERATIONAL_TIMEOUT 60
59 #define DEFAULT_RESTART_CMD WATCHFRR_SH_PATH " restart %s"
60 #define DEFAULT_START_CMD WATCHFRR_SH_PATH " start %s"
61 #define DEFAULT_STOP_CMD WATCHFRR_SH_PATH " stop %s"
63 #define PING_TOKEN "PING"
65 DEFINE_MGROUP(WATCHFRR
, "watchfrr");
66 DEFINE_MTYPE_STATIC(WATCHFRR
, WATCHFRR_DAEMON
, "watchfrr daemon entry");
68 /* Needs to be global, referenced somewhere inside libfrr. */
69 struct thread_master
*master
;
71 static bool watch_only
= false;
72 const char *pathspace
;
79 PHASE_ZEBRA_RESTART_PENDING
,
80 PHASE_WAITING_ZEBRA_UP
83 static const char *const phase_str
[] = {
87 "Waiting for other daemons to come down",
88 "Zebra restart job running",
89 "Waiting for zebra to come up",
93 #define PHASE_TIMEOUT (3*gs.restart_timeout)
94 #define STARTUP_TIMEOUT 55 * 1000
102 struct thread
*t_kill
;
106 static struct global_state
{
107 enum restart_phase phase
;
108 struct thread
*t_phase_hanging
;
109 struct thread
*t_startup_timeout
;
110 struct thread
*t_operational
;
114 long restart_timeout
;
115 long min_restart_interval
;
116 long max_restart_interval
;
117 long operational_timeout
;
118 struct daemon
*daemons
;
119 const char *restart_command
;
120 const char *start_command
;
121 const char *stop_command
;
122 struct restart_info restart
;
124 struct daemon
*special
; /* points to zebra when doing phased restart */
127 int numdown
; /* # of daemons that are not UP or UNRESPONSIVE */
130 .vtydir
= frr_vtydir
,
131 .period
= 1000 * DEFAULT_PERIOD
,
132 .timeout
= DEFAULT_TIMEOUT
,
133 .restart_timeout
= DEFAULT_RESTART_TIMEOUT
,
134 .loglevel
= DEFAULT_LOGLEVEL
,
135 .min_restart_interval
= DEFAULT_MIN_RESTART
,
136 .max_restart_interval
= DEFAULT_MAX_RESTART
,
137 .operational_timeout
= DEFAULT_OPERATIONAL_TIMEOUT
,
138 .restart_command
= DEFAULT_RESTART_CMD
,
139 .start_command
= DEFAULT_START_CMD
,
140 .stop_command
= DEFAULT_STOP_CMD
,
152 (((DMN)->state == DAEMON_UP) || ((DMN)->state == DAEMON_UNRESPONSIVE))
154 static const char *const state_str
[] = {
155 "Init", "Down", "Connecting", "Up", "Unresponsive",
160 enum daemon_state state
;
162 struct timeval echo_sent
;
163 unsigned int connect_tries
;
164 struct thread
*t_wakeup
;
165 struct thread
*t_read
;
166 struct thread
*t_write
;
168 struct restart_info restart
;
171 * For a given daemon, if we've turned on ignore timeouts
172 * ignore the timeout value and assume everything is ok
173 * This is for daemon debugging w/ gdb after we have started
174 * FRR and realize we have something that needs to be looked
180 #define OPTION_MINRESTART 2000
181 #define OPTION_MAXRESTART 2001
182 #define OPTION_DRY 2002
183 #define OPTION_NETNS 2003
184 #define OPTION_MAXOPERATIONAL 2004
186 static const struct option longopts
[] = {
187 {"daemon", no_argument
, NULL
, 'd'},
188 {"statedir", required_argument
, NULL
, 'S'},
189 {"loglevel", required_argument
, NULL
, 'l'},
190 {"interval", required_argument
, NULL
, 'i'},
191 {"timeout", required_argument
, NULL
, 't'},
192 {"restart-timeout", required_argument
, NULL
, 'T'},
193 {"restart", required_argument
, NULL
, 'r'},
194 {"start-command", required_argument
, NULL
, 's'},
195 {"kill-command", required_argument
, NULL
, 'k'},
196 {"dry", no_argument
, NULL
, OPTION_DRY
},
197 {"min-restart-interval", required_argument
, NULL
, OPTION_MINRESTART
},
198 {"max-restart-interval", required_argument
, NULL
, OPTION_MAXRESTART
},
199 {"operational-timeout", required_argument
, NULL
, OPTION_MAXOPERATIONAL
},
200 {"pid-file", required_argument
, NULL
, 'p'},
201 {"blank-string", required_argument
, NULL
, 'b'},
203 {"netns", optional_argument
, NULL
, OPTION_NETNS
},
205 {"help", no_argument
, NULL
, 'h'},
206 {"version", no_argument
, NULL
, 'v'},
209 static int try_connect(struct daemon
*dmn
);
210 static void wakeup_send_echo(struct thread
*t_wakeup
);
211 static void try_restart(struct daemon
*dmn
);
212 static void phase_check(void);
213 static void restart_done(struct daemon
*dmn
);
215 static const char *progname
;
217 void watchfrr_set_ignore_daemon(struct vty
*vty
, const char *dname
, bool ignore
)
221 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
222 if (strncmp(dmn
->name
, dname
, strlen(dmn
->name
)) == 0)
227 dmn
->ignore_timeout
= ignore
;
228 vty_out(vty
, "%s switching to %s\n", dmn
->name
,
229 ignore
? "ignore" : "watch");
231 vty_out(vty
, "%s is not configured for running at the moment",
235 static void printhelp(FILE *target
)
238 "Usage : %s [OPTION...] <daemon name> ...\n\n\
239 Watchdog program to monitor status of frr daemons and try to restart\n\
240 them if they are down or unresponsive. It determines whether a daemon is\n\
241 up based on whether it can connect to the daemon's vty unix stream socket.\n\
242 It then repeatedly sends echo commands over that socket to determine whether\n\
243 the daemon is responsive. If the daemon crashes, we will receive an EOF\n\
244 on the socket connection and know immediately that the daemon is down.\n\n\
245 The daemons to be monitored should be listed on the command line.\n\n\
246 In order to avoid attempting to restart the daemons in a fast loop,\n\
247 the -m and -M options allow you to control the minimum delay between\n\
248 restart commands. The minimum restart delay is recalculated each time\n\
249 a restart is attempted: if the time since the last restart attempt exceeds\n\
250 twice the -M value, then the restart delay is set to the -m value.\n\
251 Otherwise, the interval is doubled (but capped at the -M value).\n\n",
256 -d, --daemon Run in daemon mode. In this mode, error messages are sent\n\
257 to syslog instead of stdout.\n\
258 -S, --statedir Set the vty socket directory (default is %s)\n\
259 -N, --pathspace Insert prefix into config & socket paths\n"
261 " --netns Create and/or use Linux network namespace. If no name is\n"
262 " given, uses the value from `-N`.\n"
264 "-l, --loglevel Set the logging level (default is %d).\n\
265 The value should range from %d (LOG_EMERG) to %d (LOG_DEBUG),\n\
266 but it can be set higher than %d if extra-verbose debugging\n\
267 messages are desired.\n\
268 --min-restart-interval\n\
269 Set the minimum seconds to wait between invocations of daemon\n\
270 restart commands (default is %d).\n\
271 --max-restart-interval\n\
272 Set the maximum seconds to wait between invocations of daemon\n\
273 restart commands (default is %d).\n\
274 --operational-timeout\n\
275 Set the time before systemd is notified that we are considered\n\
276 operational again after a daemon restart (default is %d).\n\
277 -i, --interval Set the status polling interval in seconds (default is %d)\n\
278 -t, --timeout Set the unresponsiveness timeout in seconds (default is %d)\n\
279 -T, --restart-timeout\n\
280 Set the restart (kill) timeout in seconds (default is %d).\n\
281 If any background jobs are still running after this much\n\
282 time has elapsed, they will be killed.\n\
283 -r, --restart Supply a Bourne shell command to use to restart a single\n\
284 daemon. The command string should include '%%s' where the\n\
285 name of the daemon should be substituted.\n\
287 -s, --start-command\n\
288 Supply a Bourne shell to command to use to start a single\n\
289 daemon. The command string should include '%%s' where the\n\
290 name of the daemon should be substituted.\n\
292 -k, --kill-command\n\
293 Supply a Bourne shell to command to use to stop a single\n\
294 daemon. The command string should include '%%s' where the\n\
295 name of the daemon should be substituted.\n\
297 --dry Do not start or restart anything, just log.\n\
298 -p, --pid-file Set process identifier file name\n\
299 (default is %s/watchfrr.pid).\n\
300 -b, --blank-string\n\
301 When the supplied argument string is found in any of the\n\
302 various shell command arguments (-r, -s, or -k), replace\n\
303 it with a space. This is an ugly hack to circumvent problems\n\
304 passing command-line arguments with embedded spaces.\n\
305 -v, --version Print program version\n\
306 -h, --help Display this help and exit\n",
307 frr_vtydir
, DEFAULT_LOGLEVEL
, LOG_EMERG
, LOG_DEBUG
, LOG_DEBUG
,
308 DEFAULT_MIN_RESTART
, DEFAULT_MAX_RESTART
,
309 DEFAULT_OPERATIONAL_TIMEOUT
, DEFAULT_PERIOD
, DEFAULT_TIMEOUT
,
310 DEFAULT_RESTART_TIMEOUT
, DEFAULT_RESTART_CMD
, DEFAULT_START_CMD
,
311 DEFAULT_STOP_CMD
, frr_vtydir
);
314 static pid_t
run_background(char *shell_cmd
)
318 switch (child
= fork()) {
320 flog_err_sys(EC_LIB_SYSTEM_CALL
,
321 "fork failed, cannot run command [%s]: %s",
322 shell_cmd
, safe_strerror(errno
));
326 /* Use separate process group so child processes can be killed
328 if (setpgid(0, 0) < 0)
329 zlog_warn("setpgid(0,0) failed: %s",
330 safe_strerror(errno
));
334 char *const argv
[4] = {shell
, dashc
, shell_cmd
, NULL
};
335 execv("/bin/sh", argv
);
336 flog_err_sys(EC_LIB_SYSTEM_CALL
,
337 "execv(/bin/sh -c '%s') failed: %s",
338 shell_cmd
, safe_strerror(errno
));
342 /* Parent process: we will reap the child later. */
343 zlog_info("Forked background command [pid %d]: %s", (int)child
,
349 static struct timeval
*time_elapsed(struct timeval
*result
,
350 const struct timeval
*start_time
)
352 gettimeofday(result
, NULL
);
353 result
->tv_sec
-= start_time
->tv_sec
;
354 result
->tv_usec
-= start_time
->tv_usec
;
355 while (result
->tv_usec
< 0) {
356 result
->tv_usec
+= 1000000L;
362 static void restart_kill(struct thread
*t_kill
)
364 struct restart_info
*restart
= THREAD_ARG(t_kill
);
365 struct timeval delay
;
367 time_elapsed(&delay
, &restart
->time
);
369 "%s %s child process %d still running after %ld seconds, sending signal %d",
370 restart
->what
, restart
->name
, (int)restart
->pid
,
371 (long)delay
.tv_sec
, (restart
->kills
? SIGKILL
: SIGTERM
));
372 kill(-restart
->pid
, (restart
->kills
? SIGKILL
: SIGTERM
));
374 thread_add_timer(master
, restart_kill
, restart
, gs
.restart_timeout
,
378 static struct restart_info
*find_child(pid_t child
)
381 if (gs
.restart
.pid
== child
)
384 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
385 if (dmn
->restart
.pid
== child
)
386 return &dmn
->restart
;
391 static void sigchild(void)
397 struct restart_info
*restart
;
400 switch (child
= waitpid(-1, &status
, WNOHANG
)) {
402 flog_err_sys(EC_LIB_SYSTEM_CALL
, "waitpid failed: %s",
403 safe_strerror(errno
));
406 zlog_warn("SIGCHLD received, but waitpid did not reap a child");
410 if (child
== integrated_write_pid
) {
411 integrated_write_sigchld(status
);
415 if ((restart
= find_child(child
)) != NULL
) {
416 name
= restart
->name
;
417 what
= restart
->what
;
420 thread_cancel(&restart
->t_kill
);
422 /* Update restart time to reflect the time the command
424 gettimeofday(&restart
->time
, NULL
);
428 "waitpid returned status for an unknown child process %d",
433 if (WIFSTOPPED(status
))
434 zlog_warn("%s %s process %d is stopped", what
, name
,
436 else if (WIFSIGNALED(status
))
437 zlog_warn("%s %s process %d terminated due to signal %d", what
,
438 name
, (int)child
, WTERMSIG(status
));
439 else if (WIFEXITED(status
)) {
440 if (WEXITSTATUS(status
) != 0)
442 "%s %s process %d exited with non-zero status %d",
443 what
, name
, (int)child
, WEXITSTATUS(status
));
445 zlog_debug("%s %s process %d exited normally", what
,
448 if (restart
&& restart
!= &gs
.restart
) {
449 dmn
= container_of(restart
, struct daemon
,
453 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
459 "cannot interpret %s %s process %d wait status 0x%x",
460 what
, name
, (int)child
, status
);
464 static int run_job(struct restart_info
*restart
, const char *cmdtype
,
465 const char *command
, int force
, int update_interval
)
467 struct timeval delay
;
469 if (gs
.loglevel
> LOG_DEBUG
+ 1)
470 zlog_debug("attempting to %s %s", cmdtype
, restart
->name
);
473 if (gs
.loglevel
> LOG_DEBUG
+ 1)
475 "cannot %s %s, previous pid %d still running",
476 cmdtype
, restart
->name
, (int)restart
->pid
);
482 snprintf(buffer
, sizeof(buffer
), "restarting %s", restart
->name
);
483 systemd_send_status(buffer
);
485 /* Note: time_elapsed test must come before the force test, since we
487 to make sure that delay is initialized for use below in updating the
489 if ((time_elapsed(&delay
, &restart
->time
)->tv_sec
< restart
->interval
)
492 if (gs
.loglevel
> LOG_DEBUG
+ 1)
494 "postponing %s %s: elapsed time %ld < retry interval %ld",
495 cmdtype
, restart
->name
, (long)delay
.tv_sec
,
500 gettimeofday(&restart
->time
, NULL
);
503 char cmd
[strlen(command
) + strlen(restart
->name
) + 1];
504 snprintf(cmd
, sizeof(cmd
), command
, restart
->name
);
505 if ((restart
->pid
= run_background(cmd
)) > 0) {
506 thread_add_timer(master
, restart_kill
, restart
,
507 gs
.restart_timeout
, &restart
->t_kill
);
508 restart
->what
= cmdtype
;
514 /* Calculate the new restart interval. */
515 if (update_interval
) {
516 if (delay
.tv_sec
> 2 * gs
.max_restart_interval
)
517 restart
->interval
= gs
.min_restart_interval
;
518 else if ((restart
->interval
*= 2) > gs
.max_restart_interval
)
519 restart
->interval
= gs
.max_restart_interval
;
520 if (gs
.loglevel
> LOG_DEBUG
+ 1)
521 zlog_debug("restart %s interval is now %ld",
522 restart
->name
, restart
->interval
);
527 #define SET_READ_HANDLER(DMN) \
529 (DMN)->t_read = NULL; \
530 thread_add_read(master, handle_read, (DMN), (DMN)->fd, \
534 #define SET_WAKEUP_DOWN(DMN) \
536 (DMN)->t_wakeup = NULL; \
537 thread_add_timer_msec(master, wakeup_down, (DMN), \
538 FUZZY(gs.period), &(DMN)->t_wakeup); \
541 #define SET_WAKEUP_UNRESPONSIVE(DMN) \
543 (DMN)->t_wakeup = NULL; \
544 thread_add_timer_msec(master, wakeup_unresponsive, (DMN), \
545 FUZZY(gs.period), &(DMN)->t_wakeup); \
548 #define SET_WAKEUP_ECHO(DMN) \
550 (DMN)->t_wakeup = NULL; \
551 thread_add_timer_msec(master, wakeup_send_echo, (DMN), \
552 FUZZY(gs.period), &(DMN)->t_wakeup); \
555 static void wakeup_down(struct thread
*t_wakeup
)
557 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
559 dmn
->t_wakeup
= NULL
;
560 if (try_connect(dmn
) < 0)
561 SET_WAKEUP_DOWN(dmn
);
562 if ((dmn
->connect_tries
> 1) && (dmn
->state
!= DAEMON_UP
))
566 static void wakeup_init(struct thread
*t_wakeup
)
568 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
570 dmn
->t_wakeup
= NULL
;
571 if (try_connect(dmn
) < 0) {
573 "%s state -> down : initial connection attempt failed",
575 dmn
->state
= DAEMON_DOWN
;
580 static void restart_done(struct daemon
*dmn
)
582 if (dmn
->state
!= DAEMON_DOWN
) {
584 "Daemon: %s: is in %s state but expected it to be in DAEMON_DOWN state",
585 dmn
->name
, state_str
[dmn
->state
]);
588 THREAD_OFF(dmn
->t_wakeup
);
590 if (try_connect(dmn
) < 0)
591 SET_WAKEUP_DOWN(dmn
);
594 static void daemon_restarting_operational(struct thread
*thread
)
596 systemd_send_status("FRR Operational");
599 static void daemon_down(struct daemon
*dmn
, const char *why
)
601 if (IS_UP(dmn
) || (dmn
->state
== DAEMON_INIT
))
602 flog_err(EC_WATCHFRR_CONNECTION
, "%s state -> down : %s",
604 else if (gs
.loglevel
> LOG_DEBUG
)
605 zlog_debug("%s still down : %s", dmn
->name
, why
);
608 dmn
->state
= DAEMON_DOWN
;
613 THREAD_OFF(dmn
->t_read
);
614 THREAD_OFF(dmn
->t_write
);
615 THREAD_OFF(dmn
->t_wakeup
);
616 if (try_connect(dmn
) < 0)
617 SET_WAKEUP_DOWN(dmn
);
619 systemd_send_status("FRR partially operational");
623 static void handle_read(struct thread
*t_read
)
625 struct daemon
*dmn
= THREAD_ARG(t_read
);
626 static const char resp
[sizeof(PING_TOKEN
) + 4] = PING_TOKEN
"\n";
627 char buf
[sizeof(resp
) + 100];
629 struct timeval delay
;
632 if ((rc
= read(dmn
->fd
, buf
, sizeof(buf
))) < 0) {
635 if (ERRNO_IO_RETRY(errno
)) {
636 /* Pretend it never happened. */
637 SET_READ_HANDLER(dmn
);
640 snprintf(why
, sizeof(why
), "unexpected read error: %s",
641 safe_strerror(errno
));
642 daemon_down(dmn
, why
);
646 daemon_down(dmn
, "read returned EOF");
649 if (!dmn
->echo_sent
.tv_sec
) {
650 char why
[sizeof(buf
) + 100];
651 snprintf(why
, sizeof(why
),
652 "unexpected read returns %d bytes: %.*s", (int)rc
,
654 daemon_down(dmn
, why
);
658 /* We are expecting an echo response: is there any chance that the
659 response would not be returned entirely in the first read? That
660 seems inconceivable... */
661 if ((rc
!= sizeof(resp
)) || memcmp(buf
, resp
, sizeof(resp
))) {
662 char why
[100 + sizeof(buf
)];
663 snprintf(why
, sizeof(why
),
664 "read returned bad echo response of %d bytes (expecting %u): %.*s",
665 (int)rc
, (unsigned int)sizeof(resp
), (int)rc
, buf
);
666 daemon_down(dmn
, why
);
670 time_elapsed(&delay
, &dmn
->echo_sent
);
671 dmn
->echo_sent
.tv_sec
= 0;
672 if (dmn
->state
== DAEMON_UNRESPONSIVE
) {
673 if (delay
.tv_sec
< gs
.timeout
) {
674 dmn
->state
= DAEMON_UP
;
676 "%s state -> up : echo response received after %ld.%06ld seconds",
677 dmn
->name
, (long)delay
.tv_sec
,
678 (long)delay
.tv_usec
);
681 "%s: slow echo response finally received after %ld.%06ld seconds",
682 dmn
->name
, (long)delay
.tv_sec
,
683 (long)delay
.tv_usec
);
684 } else if (gs
.loglevel
> LOG_DEBUG
+ 1)
685 zlog_debug("%s: echo response received after %ld.%06ld seconds",
686 dmn
->name
, (long)delay
.tv_sec
, (long)delay
.tv_usec
);
688 SET_READ_HANDLER(dmn
);
689 thread_cancel(&dmn
->t_wakeup
);
690 SET_WAKEUP_ECHO(dmn
);
694 * Wait till we notice that all daemons are ready before
695 * we send we are ready to systemd
697 static void daemon_send_ready(int exitcode
)
707 zlog_notice("all daemons up, doing startup-complete notify");
708 else if (gs
.numdown
< gs
.numdaemons
)
709 flog_err(EC_WATCHFRR_CONNECTION
,
710 "startup did not complete within timeout (%d/%d daemons running)",
711 gs
.numdaemons
- gs
.numdown
, gs
.numdaemons
);
713 flog_err(EC_WATCHFRR_CONNECTION
,
714 "all configured daemons failed to start -- exiting watchfrr");
721 snprintf(started
, sizeof(started
), "%s/%s", frr_vtydir
,
723 fp
= fopen(started
, "w");
727 systemd_send_started(master
);
728 systemd_send_status("FRR Operational");
732 static void daemon_up(struct daemon
*dmn
, const char *why
)
734 dmn
->state
= DAEMON_UP
;
736 dmn
->connect_tries
= 0;
737 zlog_notice("%s state -> up : %s", dmn
->name
, why
);
738 if (gs
.numdown
== 0) {
739 daemon_send_ready(0);
741 THREAD_OFF(gs
.t_operational
);
743 thread_add_timer(master
, daemon_restarting_operational
, NULL
,
744 gs
.operational_timeout
, &gs
.t_operational
);
747 SET_WAKEUP_ECHO(dmn
);
751 static void check_connect(struct thread
*t_write
)
753 struct daemon
*dmn
= THREAD_ARG(t_write
);
755 socklen_t reslen
= sizeof(sockerr
);
758 if (getsockopt(dmn
->fd
, SOL_SOCKET
, SO_ERROR
, (char *)&sockerr
, &reslen
)
760 zlog_warn("%s: check_connect: getsockopt failed: %s", dmn
->name
,
761 safe_strerror(errno
));
763 "getsockopt failed checking connection success");
766 if ((reslen
== sizeof(sockerr
)) && sockerr
) {
770 "getsockopt reports that connection attempt failed: %s",
771 safe_strerror(sockerr
));
772 daemon_down(dmn
, why
);
776 daemon_up(dmn
, "delayed connect succeeded");
779 static void wakeup_connect_hanging(struct thread
*t_wakeup
)
781 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
784 dmn
->t_wakeup
= NULL
;
785 snprintf(why
, sizeof(why
),
786 "connection attempt timed out after %ld seconds", gs
.timeout
);
787 daemon_down(dmn
, why
);
790 /* Making connection to protocol daemon. */
791 static int try_connect(struct daemon
*dmn
)
794 struct sockaddr_un addr
;
797 if (gs
.loglevel
> LOG_DEBUG
+ 1)
798 zlog_debug("%s: attempting to connect", dmn
->name
);
799 dmn
->connect_tries
++;
801 memset(&addr
, 0, sizeof(addr
));
802 addr
.sun_family
= AF_UNIX
;
803 snprintf(addr
.sun_path
, sizeof(addr
.sun_path
), "%s/%s.vty", gs
.vtydir
,
805 #ifdef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN
806 len
= addr
.sun_len
= SUN_LEN(&addr
);
808 len
= sizeof(addr
.sun_family
) + strlen(addr
.sun_path
);
809 #endif /* HAVE_STRUCT_SOCKADDR_UN_SUN_LEN */
811 /* Quick check to see if we might succeed before we go to the trouble
812 of creating a socket. */
813 if (access(addr
.sun_path
, W_OK
) < 0) {
815 flog_err_sys(EC_LIB_SYSTEM_CALL
,
816 "%s: access to socket %s denied: %s",
817 dmn
->name
, addr
.sun_path
,
818 safe_strerror(errno
));
822 if ((sock
= socket(AF_UNIX
, SOCK_STREAM
, 0)) < 0) {
823 flog_err_sys(EC_LIB_SOCKET
, "%s(%s): cannot make socket: %s",
824 __func__
, addr
.sun_path
, safe_strerror(errno
));
828 if (set_nonblocking(sock
) < 0 || set_cloexec(sock
) < 0) {
829 flog_err_sys(EC_LIB_SYSTEM_CALL
,
830 "%s(%s): set_nonblocking/cloexec(%d) failed",
831 __func__
, addr
.sun_path
, sock
);
836 if (connect(sock
, (struct sockaddr
*)&addr
, len
) < 0) {
837 if ((errno
!= EINPROGRESS
) && (errno
!= EWOULDBLOCK
)) {
838 if (gs
.loglevel
> LOG_DEBUG
)
839 zlog_debug("%s(%s): connect failed: %s",
840 __func__
, addr
.sun_path
,
841 safe_strerror(errno
));
845 if (gs
.loglevel
> LOG_DEBUG
)
846 zlog_debug("%s: connection in progress", dmn
->name
);
847 dmn
->state
= DAEMON_CONNECTING
;
849 thread_add_write(master
, check_connect
, dmn
, dmn
->fd
,
851 thread_add_timer(master
, wakeup_connect_hanging
, dmn
,
852 gs
.timeout
, &dmn
->t_wakeup
);
853 SET_READ_HANDLER(dmn
);
858 SET_READ_HANDLER(dmn
);
859 daemon_up(dmn
, "connect succeeded");
863 static void phase_hanging(struct thread
*t_hanging
)
865 gs
.t_phase_hanging
= NULL
;
866 flog_err(EC_WATCHFRR_CONNECTION
,
867 "Phase [%s] hanging for %ld seconds, aborting phased restart",
868 phase_str
[gs
.phase
], PHASE_TIMEOUT
);
869 gs
.phase
= PHASE_NONE
;
872 static void set_phase(enum restart_phase new_phase
)
874 gs
.phase
= new_phase
;
875 thread_cancel(&gs
.t_phase_hanging
);
877 thread_add_timer(master
, phase_hanging
, NULL
, PHASE_TIMEOUT
,
878 &gs
.t_phase_hanging
);
881 static void phase_check(void)
890 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
891 if (dmn
->state
== DAEMON_INIT
)
894 /* startup complete, everything out of INIT */
895 gs
.phase
= PHASE_NONE
;
896 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
897 if (dmn
->state
== DAEMON_DOWN
) {
898 SET_WAKEUP_DOWN(dmn
);
902 case PHASE_STOPS_PENDING
:
906 "Phased restart: all routing daemon stop jobs have completed.");
907 set_phase(PHASE_WAITING_DOWN
);
910 case PHASE_WAITING_DOWN
:
911 if (gs
.numdown
+ IS_UP(gs
.special
) < gs
.numdaemons
)
913 systemd_send_status("Phased Restart");
914 zlog_info("Phased restart: all routing daemons now down.");
915 run_job(&gs
.special
->restart
, "restart", gs
.restart_command
, 1,
917 set_phase(PHASE_ZEBRA_RESTART_PENDING
);
920 case PHASE_ZEBRA_RESTART_PENDING
:
921 if (gs
.special
->restart
.pid
)
923 systemd_send_status("Zebra Restarting");
924 zlog_info("Phased restart: %s restart job completed.",
926 set_phase(PHASE_WAITING_ZEBRA_UP
);
929 case PHASE_WAITING_ZEBRA_UP
:
930 if (!IS_UP(gs
.special
))
932 zlog_info("Phased restart: %s is now up.", gs
.special
->name
);
935 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
936 if (dmn
!= gs
.special
)
937 run_job(&dmn
->restart
, "start",
938 gs
.start_command
, 1, 0);
941 gs
.phase
= PHASE_NONE
;
942 THREAD_OFF(gs
.t_phase_hanging
);
943 zlog_notice("Phased global restart has completed.");
948 static void try_restart(struct daemon
*dmn
)
953 if (dmn
!= gs
.special
) {
954 if ((gs
.special
->state
== DAEMON_UP
)
955 && (gs
.phase
== PHASE_NONE
))
956 run_job(&dmn
->restart
, "restart", gs
.restart_command
, 0,
960 "%s: postponing restart attempt because master %s daemon not up [%s], or phased restart in progress",
961 dmn
->name
, gs
.special
->name
,
962 state_str
[gs
.special
->state
]);
966 if ((gs
.phase
!= PHASE_NONE
) || gs
.numpids
) {
967 if (gs
.loglevel
> LOG_DEBUG
+ 1)
969 "postponing phased global restart: restart already in progress [%s], or outstanding child processes [%d]",
970 phase_str
[gs
.phase
], gs
.numpids
);
973 /* Is it too soon for a restart? */
975 struct timeval delay
;
976 if (time_elapsed(&delay
, &gs
.special
->restart
.time
)->tv_sec
977 < gs
.special
->restart
.interval
) {
978 if (gs
.loglevel
> LOG_DEBUG
+ 1)
980 "postponing phased global restart: elapsed time %ld < retry interval %ld",
982 gs
.special
->restart
.interval
);
986 run_job(&gs
.restart
, "restart", gs
.restart_command
, 0, 1);
989 static void wakeup_unresponsive(struct thread
*t_wakeup
)
991 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
993 dmn
->t_wakeup
= NULL
;
994 if (dmn
->state
!= DAEMON_UNRESPONSIVE
)
995 flog_err(EC_WATCHFRR_CONNECTION
,
996 "%s: no longer unresponsive (now %s), wakeup should have been cancelled!",
997 dmn
->name
, state_str
[dmn
->state
]);
999 SET_WAKEUP_UNRESPONSIVE(dmn
);
1004 static void wakeup_no_answer(struct thread
*t_wakeup
)
1006 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
1008 dmn
->t_wakeup
= NULL
;
1009 dmn
->state
= DAEMON_UNRESPONSIVE
;
1010 if (dmn
->ignore_timeout
)
1012 flog_err(EC_WATCHFRR_CONNECTION
,
1013 "%s state -> unresponsive : no response yet to ping sent %ld seconds ago",
1014 dmn
->name
, gs
.timeout
);
1015 SET_WAKEUP_UNRESPONSIVE(dmn
);
1019 static void wakeup_send_echo(struct thread
*t_wakeup
)
1021 static const char echocmd
[] = "echo " PING_TOKEN
;
1023 struct daemon
*dmn
= THREAD_ARG(t_wakeup
);
1025 dmn
->t_wakeup
= NULL
;
1026 if (((rc
= write(dmn
->fd
, echocmd
, sizeof(echocmd
))) < 0)
1027 || ((size_t)rc
!= sizeof(echocmd
))) {
1028 char why
[100 + sizeof(echocmd
)];
1029 snprintf(why
, sizeof(why
),
1030 "write '%s' returned %d instead of %u", echocmd
,
1031 (int)rc
, (unsigned int)sizeof(echocmd
));
1032 daemon_down(dmn
, why
);
1034 gettimeofday(&dmn
->echo_sent
, NULL
);
1035 thread_add_timer(master
, wakeup_no_answer
, dmn
, gs
.timeout
,
1040 bool check_all_up(void)
1044 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
)
1045 if (dmn
->state
!= DAEMON_UP
)
1050 void watchfrr_status(struct vty
*vty
)
1053 struct timeval delay
;
1055 vty_out(vty
, "watchfrr global phase: %s\n", phase_str
[gs
.phase
]);
1056 vty_out(vty
, " Restart Command: %pSQq\n", gs
.restart_command
);
1057 vty_out(vty
, " Start Command: %pSQq\n", gs
.start_command
);
1058 vty_out(vty
, " Stop Command: %pSQq\n", gs
.stop_command
);
1059 vty_out(vty
, " Min Restart Interval: %ld\n", gs
.min_restart_interval
);
1060 vty_out(vty
, " Max Restart Interval: %ld\n", gs
.max_restart_interval
);
1061 vty_out(vty
, " Restart Timeout: %ld\n", gs
.restart_timeout
);
1063 vty_out(vty
, " global restart running, pid %ld\n",
1064 (long)gs
.restart
.pid
);
1066 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
1067 vty_out(vty
, " %-20s %s%s", dmn
->name
, state_str
[dmn
->state
],
1068 dmn
->ignore_timeout
? "/Ignoring Timeout\n" : "\n");
1069 if (dmn
->restart
.pid
)
1070 vty_out(vty
, " restart running, pid %ld\n",
1071 (long)dmn
->restart
.pid
);
1072 else if (dmn
->state
== DAEMON_DOWN
&&
1073 time_elapsed(&delay
, &dmn
->restart
.time
)->tv_sec
1074 < dmn
->restart
.interval
)
1075 vty_out(vty
, " restarting in %jd seconds (%jds backoff interval)\n",
1076 (intmax_t)dmn
->restart
.interval
1077 - (intmax_t)delay
.tv_sec
,
1078 (intmax_t)dmn
->restart
.interval
);
1082 static void sigint(void)
1084 zlog_notice("Terminating on signal");
1085 systemd_send_stopping();
1089 static int valid_command(const char *cmd
)
1096 return ((p
= strchr(cmd
, '%')) != NULL
) && (*(p
+ 1) == 's')
1097 && !strchr(p
+ 1, '%');
1100 /* This is an ugly hack to circumvent problems with passing command-line
1101 arguments that contain spaces. The fix is to use a configuration file. */
1102 static char *translate_blanks(const char *cmd
, const char *blankstr
)
1106 size_t bslen
= strlen(blankstr
);
1108 if (!(res
= strdup(cmd
))) {
1112 while ((p
= strstr(res
, blankstr
)) != NULL
) {
1115 memmove(p
+ 1, p
+ bslen
, strlen(p
+ bslen
) + 1);
1120 static void startup_timeout(struct thread
*t_wakeup
)
1122 daemon_send_ready(1);
1127 #include <sys/mount.h>
1130 #define NETNS_RUN_DIR "/var/run/netns"
1132 static void netns_create(int dirfd
, const char *nsname
)
1134 /* make /var/run/netns shared between mount namespaces
1135 * just like iproute2 sets it up
1137 if (mount("", NETNS_RUN_DIR
, "none", MS_SHARED
| MS_REC
, NULL
)) {
1138 if (errno
!= EINVAL
) {
1143 if (mount(NETNS_RUN_DIR
, NETNS_RUN_DIR
, "none",
1144 MS_BIND
| MS_REC
, NULL
)) {
1149 if (mount("", NETNS_RUN_DIR
, "none", MS_SHARED
| MS_REC
,
1156 /* need an empty file to mount on top of */
1157 int nsfd
= openat(dirfd
, nsname
, O_CREAT
| O_RDONLY
| O_EXCL
, 0);
1160 fprintf(stderr
, "failed to create \"%s/%s\": %s\n",
1161 NETNS_RUN_DIR
, nsname
, strerror(errno
));
1166 if (unshare(CLONE_NEWNET
)) {
1168 unlinkat(dirfd
, nsname
, 0);
1172 char *dstpath
= asprintfrr(MTYPE_TMP
, "%s/%s", NETNS_RUN_DIR
, nsname
);
1174 /* bind-mount so the namespace has a name and is persistent */
1175 if (mount("/proc/self/ns/net", dstpath
, "none", MS_BIND
, NULL
) < 0) {
1176 fprintf(stderr
, "failed to bind-mount netns to \"%s\": %s\n",
1177 dstpath
, strerror(errno
));
1178 unlinkat(dirfd
, nsname
, 0);
1182 XFREE(MTYPE_TMP
, dstpath
);
1185 static void netns_setup(const char *nsname
)
1189 dirfd
= open(NETNS_RUN_DIR
, O_DIRECTORY
| O_RDONLY
);
1191 if (errno
== ENOTDIR
) {
1192 fprintf(stderr
, "error: \"%s\" is not a directory!\n",
1195 } else if (errno
== ENOENT
) {
1196 if (mkdir(NETNS_RUN_DIR
, 0755)) {
1197 fprintf(stderr
, "error: \"%s\": mkdir: %s\n",
1198 NETNS_RUN_DIR
, strerror(errno
));
1201 dirfd
= open(NETNS_RUN_DIR
, O_DIRECTORY
| O_RDONLY
);
1203 fprintf(stderr
, "error: \"%s\": opendir: %s\n",
1204 NETNS_RUN_DIR
, strerror(errno
));
1208 fprintf(stderr
, "error: \"%s\": %s\n",
1209 NETNS_RUN_DIR
, strerror(errno
));
1214 nsfd
= openat(dirfd
, nsname
, O_RDONLY
);
1215 if (nsfd
< 0 && errno
!= ENOENT
) {
1216 fprintf(stderr
, "error: \"%s/%s\": %s\n",
1217 NETNS_RUN_DIR
, nsname
, strerror(errno
));
1221 netns_create(dirfd
, nsname
);
1223 if (setns(nsfd
, CLONE_NEWNET
)) {
1231 /* make sure loopback is up... weird things happen otherwise.
1232 * ioctl is perfectly fine for this, don't need netlink...
1235 struct ifreq ifr
= { };
1237 strlcpy(ifr
.ifr_name
, "lo", sizeof(ifr
.ifr_name
));
1239 sockfd
= socket(AF_INET
, SOCK_DGRAM
, 0);
1244 if (ioctl(sockfd
, SIOCGIFFLAGS
, &ifr
)) {
1245 perror("ioctl(SIOCGIFFLAGS, \"lo\")");
1248 if (!(ifr
.ifr_flags
& IFF_UP
)) {
1249 ifr
.ifr_flags
|= IFF_UP
;
1250 if (ioctl(sockfd
, SIOCSIFFLAGS
, &ifr
)) {
1251 perror("ioctl(SIOCSIFFLAGS, \"lo\")");
1258 #else /* !GNU_LINUX */
1260 static void netns_setup(const char *nsname
)
1262 fprintf(stderr
, "network namespaces are only available on Linux\n");
1267 static void watchfrr_init(int argc
, char **argv
)
1269 const char *special
= "zebra";
1271 struct daemon
*dmn
, **add
= &gs
.daemons
;
1272 char alldaemons
[512] = "", *p
= alldaemons
;
1274 thread_add_timer_msec(master
, startup_timeout
, NULL
, STARTUP_TIMEOUT
,
1275 &gs
.t_startup_timeout
);
1277 for (i
= optind
; i
< argc
; i
++) {
1278 dmn
= XCALLOC(MTYPE_WATCHFRR_DAEMON
, sizeof(*dmn
));
1280 dmn
->name
= dmn
->restart
.name
= argv
[i
];
1281 dmn
->state
= DAEMON_INIT
;
1285 thread_add_timer_msec(master
, wakeup_init
, dmn
, 0,
1287 dmn
->restart
.interval
= gs
.min_restart_interval
;
1291 if (!strcmp(dmn
->name
, special
))
1297 "Must specify one or more daemons to monitor.\n\n");
1300 if (!watch_only
&& !gs
.special
) {
1301 fprintf(stderr
, "\"%s\" daemon must be in daemon lists\n\n",
1306 for (dmn
= gs
.daemons
; dmn
; dmn
= dmn
->next
) {
1307 snprintf(p
, alldaemons
+ sizeof(alldaemons
) - p
, "%s%s",
1308 (p
== alldaemons
) ? "" : " ", dmn
->name
);
1311 zlog_notice("%s %s watching [%s]%s", progname
, FRR_VERSION
, alldaemons
,
1312 watch_only
? ", monitor mode" : "");
1315 struct zebra_privs_t watchfrr_privs
= {
1317 .vty_group
= VTY_GROUP
,
1321 static struct frr_signal_t watchfrr_signals
[] = {
1332 .handler
= sigchild
,
1336 FRR_DAEMON_INFO(watchfrr
, WATCHFRR
,
1337 .flags
= FRR_NO_PRIVSEP
| FRR_NO_TCPVTY
| FRR_LIMITED_CLI
1338 | FRR_NO_CFG_PID_DRY
| FRR_NO_ZCLIENT
1341 .printhelp
= printhelp
,
1342 .copyright
= "Copyright 2004 Andrew J. Schorr",
1344 .signals
= watchfrr_signals
,
1345 .n_signals
= array_size(watchfrr_signals
),
1347 .privs
= &watchfrr_privs
,
1350 #define DEPRECATED_OPTIONS "aAezR:"
1352 int main(int argc
, char **argv
)
1355 const char *blankstr
= NULL
;
1356 const char *netns
= NULL
;
1357 bool netns_en
= false;
1359 frr_preinit(&watchfrr_di
, argc
, argv
);
1360 progname
= watchfrr_di
.progname
;
1362 frr_opt_add("b:di:k:l:N:p:r:S:s:t:T:" DEPRECATED_OPTIONS
, longopts
, "");
1364 gs
.restart
.name
= "all";
1365 while ((opt
= frr_getopt(argc
, argv
, NULL
)) != EOF
) {
1366 if (opt
&& opt
< 128 && strchr(DEPRECATED_OPTIONS
, opt
)) {
1368 "The -%c option no longer exists.\n"
1369 "Please refer to the watchfrr(8) man page.\n",
1384 if (!valid_command(optarg
)) {
1386 "Invalid kill command, must contain '%%s': %s\n",
1390 gs
.stop_command
= optarg
;
1394 if ((sscanf(optarg
, "%d%1s", &gs
.loglevel
, garbage
)
1396 || (gs
.loglevel
< LOG_EMERG
)) {
1398 "Invalid loglevel argument: %s\n",
1403 case OPTION_MINRESTART
: {
1405 if ((sscanf(optarg
, "%ld%1s", &gs
.min_restart_interval
,
1408 || (gs
.min_restart_interval
< 0)) {
1410 "Invalid min_restart_interval argument: %s\n",
1415 case OPTION_MAXRESTART
: {
1417 if ((sscanf(optarg
, "%ld%1s", &gs
.max_restart_interval
,
1420 || (gs
.max_restart_interval
< 0)) {
1422 "Invalid max_restart_interval argument: %s\n",
1427 case OPTION_MAXOPERATIONAL
: {
1430 if ((sscanf(optarg
, "%ld%1s", &gs
.operational_timeout
,
1432 (gs
.max_restart_interval
< 0)) {
1434 "Invalid Operational_timeout argument: %s\n",
1441 if (optarg
&& strchr(optarg
, '/')) {
1443 "invalid network namespace name \"%s\" (may not contain slashes)\n",
1452 if ((sscanf(optarg
, "%d%1s", &period
, garbage
) != 1)
1453 || (gs
.period
< 1)) {
1455 "Invalid interval argument: %s\n",
1459 gs
.period
= 1000 * period
;
1462 watchfrr_di
.pid_file
= optarg
;
1465 if (!valid_command(optarg
)) {
1467 "Invalid restart command, must contain '%%s': %s\n",
1471 gs
.restart_command
= optarg
;
1474 if (!valid_command(optarg
)) {
1476 "Invalid start command, must contain '%%s': %s\n",
1480 gs
.start_command
= optarg
;
1487 if ((sscanf(optarg
, "%ld%1s", &gs
.timeout
, garbage
)
1489 || (gs
.timeout
< 1)) {
1491 "Invalid timeout argument: %s\n",
1498 if ((sscanf(optarg
, "%ld%1s", &gs
.restart_timeout
,
1501 || (gs
.restart_timeout
< 1)) {
1503 "Invalid restart timeout argument: %s\n",
1509 fputs("Invalid option.\n", stderr
);
1515 && (gs
.start_command
|| gs
.stop_command
|| gs
.restart_command
)) {
1516 fputs("Options -r/-s/-k are not used when --dry is active.\n",
1520 && (!gs
.restart_command
|| !gs
.start_command
|| !gs
.stop_command
)) {
1522 "Options -s (start), -k (kill), and -r (restart) are required.\n");
1527 if (gs
.restart_command
)
1528 gs
.restart_command
=
1529 translate_blanks(gs
.restart_command
, blankstr
);
1530 if (gs
.start_command
)
1532 translate_blanks(gs
.start_command
, blankstr
);
1533 if (gs
.stop_command
)
1535 translate_blanks(gs
.stop_command
, blankstr
);
1538 gs
.restart
.interval
= gs
.min_restart_interval
;
1540 /* env variable for the processes that we start */
1541 if (watchfrr_di
.pathspace
)
1542 setenv("FRR_PATHSPACE", watchfrr_di
.pathspace
, 1);
1544 unsetenv("FRR_PATHSPACE");
1547 * when watchfrr_di.pathspace is read, if it is not specified
1548 * pathspace is NULL as expected
1550 pathspace
= watchfrr_di
.pathspace
;
1552 if (netns_en
&& !netns
)
1553 netns
= watchfrr_di
.pathspace
;
1555 if (netns_en
&& netns
&& netns
[0])
1558 master
= frr_init();
1559 watchfrr_error_init();
1560 watchfrr_init(argc
, argv
);
1561 watchfrr_vty_init();
1565 if (watchfrr_di
.daemon_mode
)
1566 zlog_syslog_set_prio_min(MIN(gs
.loglevel
, LOG_DEBUG
));
1568 zlog_aux_init(NULL
, MIN(gs
.loglevel
, LOG_DEBUG
));
1572 systemd_send_stopping();