]> git.proxmox.com Git - mirror_frr.git/blob - watchfrr/watchfrr.c
projects / mirror_frr.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge pull request #3606 from pguibert6WIND/if_rmap_container
[mirror_frr.git] / watchfrr / watchfrr.c
1 /*
2 * Monitor status of frr daemons and restart if necessary.
3 *
4 * Copyright (C) 2004 Andrew J. Schorr
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along
17 * with this program; see the file COPYING; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #include <zebra.h>
22 #include <thread.h>
23 #include <log.h>
24 #include <network.h>
25 #include <sigevent.h>
26 #include <lib/version.h>
27 #include "command.h"
28 #include "memory_vty.h"
29 #include "libfrr.h"
30 #include "lib_errors.h"
31
32 #include <getopt.h>
33 #include <sys/un.h>
34 #include <sys/wait.h>
35 #include <memory.h>
36 #include <systemd.h>
37
38 #include "watchfrr.h"
39 #include "watchfrr_errors.h"
40
41 #ifndef MIN
42 #define MIN(X,Y) (((X) <= (Y)) ? (X) : (Y))
43 #endif
44
45 /* Macros to help randomize timers. */
46 #define JITTER(X) ((random() % ((X)+1))-((X)/2))
47 #define FUZZY(X) ((X)+JITTER((X)/20))
48
49 #define DEFAULT_PERIOD 5
50 #define DEFAULT_TIMEOUT 90
51 #define DEFAULT_RESTART_TIMEOUT 20
52 #define DEFAULT_LOGLEVEL LOG_INFO
53 #define DEFAULT_MIN_RESTART 60
54 #define DEFAULT_MAX_RESTART 600
55
56 #define DEFAULT_RESTART_CMD WATCHFRR_SH_PATH " restart %s"
57 #define DEFAULT_START_CMD WATCHFRR_SH_PATH " start %s"
58 #define DEFAULT_STOP_CMD WATCHFRR_SH_PATH " stop %s"
59
60 #define PING_TOKEN "PING"
61
62 DEFINE_MGROUP(WATCHFRR, "watchfrr")
63 DEFINE_MTYPE_STATIC(WATCHFRR, WATCHFRR_DAEMON, "watchfrr daemon entry")
64
65 /* Needs to be global, referenced somewhere inside libfrr. */
66 struct thread_master *master;
67
68 static bool watch_only = false;
69
70 typedef enum {
71 PHASE_NONE = 0,
72 PHASE_INIT,
73 PHASE_STOPS_PENDING,
74 PHASE_WAITING_DOWN,
75 PHASE_ZEBRA_RESTART_PENDING,
76 PHASE_WAITING_ZEBRA_UP
77 } restart_phase_t;
78
79 static const char *phase_str[] = {
80 "Idle",
81 "Startup",
82 "Stop jobs running",
83 "Waiting for other daemons to come down",
84 "Zebra restart job running",
85 "Waiting for zebra to come up",
86 "Start jobs running",
87 };
88
89 #define PHASE_TIMEOUT (3*gs.restart_timeout)
90 #define STARTUP_TIMEOUT 55 * 1000
91
92 struct restart_info {
93 const char *name;
94 const char *what;
95 pid_t pid;
96 struct timeval time;
97 long interval;
98 struct thread *t_kill;
99 int kills;
100 };
101
102 static struct global_state {
103 restart_phase_t phase;
104 struct thread *t_phase_hanging;
105 struct thread *t_startup_timeout;
106 const char *vtydir;
107 long period;
108 long timeout;
109 long restart_timeout;
110 long min_restart_interval;
111 long max_restart_interval;
112 struct daemon *daemons;
113 const char *restart_command;
114 const char *start_command;
115 const char *stop_command;
116 struct restart_info restart;
117 int loglevel;
118 struct daemon *special; /* points to zebra when doing phased restart */
119 int numdaemons;
120 int numpids;
121 int numdown; /* # of daemons that are not UP or UNRESPONSIVE */
122 } gs = {
123 .phase = PHASE_INIT,
124 .vtydir = frr_vtydir,
125 .period = 1000 * DEFAULT_PERIOD,
126 .timeout = DEFAULT_TIMEOUT,
127 .restart_timeout = DEFAULT_RESTART_TIMEOUT,
128 .loglevel = DEFAULT_LOGLEVEL,
129 .min_restart_interval = DEFAULT_MIN_RESTART,
130 .max_restart_interval = DEFAULT_MAX_RESTART,
131 .restart_command = DEFAULT_RESTART_CMD,
132 .start_command = DEFAULT_START_CMD,
133 .stop_command = DEFAULT_STOP_CMD,
134 };
135
136 typedef enum {
137 DAEMON_INIT,
138 DAEMON_DOWN,
139 DAEMON_CONNECTING,
140 DAEMON_UP,
141 DAEMON_UNRESPONSIVE
142 } daemon_state_t;
143
144 #define IS_UP(DMN) \
145 (((DMN)->state == DAEMON_UP) || ((DMN)->state == DAEMON_UNRESPONSIVE))
146
147 static const char *state_str[] = {
148 "Init", "Down", "Connecting", "Up", "Unresponsive",
149 };
150
151 struct daemon {
152 const char *name;
153 daemon_state_t state;
154 int fd;
155 struct timeval echo_sent;
156 unsigned int connect_tries;
157 struct thread *t_wakeup;
158 struct thread *t_read;
159 struct thread *t_write;
160 struct daemon *next;
161 struct restart_info restart;
162 };
163
164 #define OPTION_MINRESTART 2000
165 #define OPTION_MAXRESTART 2001
166 #define OPTION_DRY 2002
167
168 static const struct option longopts[] = {
169 {"daemon", no_argument, NULL, 'd'},
170 {"statedir", required_argument, NULL, 'S'},
171 {"loglevel", required_argument, NULL, 'l'},
172 {"interval", required_argument, NULL, 'i'},
173 {"timeout", required_argument, NULL, 't'},
174 {"restart-timeout", required_argument, NULL, 'T'},
175 {"restart", required_argument, NULL, 'r'},
176 {"start-command", required_argument, NULL, 's'},
177 {"kill-command", required_argument, NULL, 'k'},
178 {"dry", no_argument, NULL, OPTION_DRY},
179 {"min-restart-interval", required_argument, NULL, OPTION_MINRESTART},
180 {"max-restart-interval", required_argument, NULL, OPTION_MAXRESTART},
181 {"pid-file", required_argument, NULL, 'p'},
182 {"blank-string", required_argument, NULL, 'b'},
183 {"help", no_argument, NULL, 'h'},
184 {"version", no_argument, NULL, 'v'},
185 {NULL, 0, NULL, 0}};
186
187 static int try_connect(struct daemon *dmn);
188 static int wakeup_send_echo(struct thread *t_wakeup);
189 static void try_restart(struct daemon *dmn);
190 static void phase_check(void);
191 static void restart_done(struct daemon *dmn);
192
193 static const char *progname;
194 static void printhelp(FILE *target)
195 {
196 fprintf(target,
197 "Usage : %s [OPTION...] <daemon name> ...\n\n\
198 Watchdog program to monitor status of frr daemons and try to restart\n\
199 them if they are down or unresponsive. It determines whether a daemon is\n\
200 up based on whether it can connect to the daemon's vty unix stream socket.\n\
201 It then repeatedly sends echo commands over that socket to determine whether\n\
202 the daemon is responsive. If the daemon crashes, we will receive an EOF\n\
203 on the socket connection and know immediately that the daemon is down.\n\n\
204 The daemons to be monitored should be listed on the command line.\n\n\
205 In order to avoid attempting to restart the daemons in a fast loop,\n\
206 the -m and -M options allow you to control the minimum delay between\n\
207 restart commands. The minimum restart delay is recalculated each time\n\
208 a restart is attempted: if the time since the last restart attempt exceeds\n\
209 twice the -M value, then the restart delay is set to the -m value.\n\
210 Otherwise, the interval is doubled (but capped at the -M value).\n\n",
211 progname);
212
213 fprintf(target,
214 "Options:\n\
215 -d, --daemon Run in daemon mode. In this mode, error messages are sent\n\
216 to syslog instead of stdout.\n\
217 -S, --statedir Set the vty socket directory (default is %s)\n\
218 -l, --loglevel Set the logging level (default is %d).\n\
219 The value should range from %d (LOG_EMERG) to %d (LOG_DEBUG),\n\
220 but it can be set higher than %d if extra-verbose debugging\n\
221 messages are desired.\n\
222 --min-restart-interval\n\
223 Set the minimum seconds to wait between invocations of daemon\n\
224 restart commands (default is %d).\n\
225 --max-restart-interval\n\
226 Set the maximum seconds to wait between invocations of daemon\n\
227 restart commands (default is %d).\n\
228 -i, --interval Set the status polling interval in seconds (default is %d)\n\
229 -t, --timeout Set the unresponsiveness timeout in seconds (default is %d)\n\
230 -T, --restart-timeout\n\
231 Set the restart (kill) timeout in seconds (default is %d).\n\
232 If any background jobs are still running after this much\n\
233 time has elapsed, they will be killed.\n\
234 -r, --restart Supply a Bourne shell command to use to restart a single\n\
235 daemon. The command string should include '%%s' where the\n\
236 name of the daemon should be substituted.\n\
237 (default: '%s')\n\
238 -s, --start-command\n\
239 Supply a Bourne shell to command to use to start a single\n\
240 daemon. The command string should include '%%s' where the\n\
241 name of the daemon should be substituted.\n\
242 (default: '%s')\n\
243 -k, --kill-command\n\
244 Supply a Bourne shell to command to use to stop a single\n\
245 daemon. The command string should include '%%s' where the\n\
246 name of the daemon should be substituted.\n\
247 (default: '%s')\n\
248 --dry Do not start or restart anything, just log.\n\
249 -p, --pid-file Set process identifier file name\n\
250 (default is %s/watchfrr.pid).\n\
251 -b, --blank-string\n\
252 When the supplied argument string is found in any of the\n\
253 various shell command arguments (-r, -s, or -k), replace\n\
254 it with a space. This is an ugly hack to circumvent problems\n\
255 passing command-line arguments with embedded spaces.\n\
256 -v, --version Print program version\n\
257 -h, --help Display this help and exit\n",
258 frr_vtydir, DEFAULT_LOGLEVEL, LOG_EMERG, LOG_DEBUG, LOG_DEBUG,
259 DEFAULT_MIN_RESTART, DEFAULT_MAX_RESTART, DEFAULT_PERIOD,
260 DEFAULT_TIMEOUT, DEFAULT_RESTART_TIMEOUT,
261 DEFAULT_RESTART_CMD, DEFAULT_START_CMD, DEFAULT_STOP_CMD,
262 frr_vtydir);
263 }
264
265 static pid_t run_background(char *shell_cmd)
266 {
267 pid_t child;
268
269 switch (child = fork()) {
270 case -1:
271 flog_err_sys(EC_LIB_SYSTEM_CALL,
272 "fork failed, cannot run command [%s]: %s",
273 shell_cmd, safe_strerror(errno));
274 return -1;
275 case 0:
276 /* Child process. */
277 /* Use separate process group so child processes can be killed
278 * easily. */
279 if (setpgid(0, 0) < 0)
280 zlog_warn("warning: setpgid(0,0) failed: %s",
281 safe_strerror(errno));
282 {
283 char shell[] = "sh";
284 char dashc[] = "-c";
285 char *const argv[4] = {shell, dashc, shell_cmd, NULL};
286 execv("/bin/sh", argv);
287 flog_err_sys(EC_LIB_SYSTEM_CALL,
288 "execv(/bin/sh -c '%s') failed: %s",
289 shell_cmd, safe_strerror(errno));
290 _exit(127);
291 }
292 default:
293 /* Parent process: we will reap the child later. */
294 flog_err_sys(EC_LIB_SYSTEM_CALL,
295 "Forked background command [pid %d]: %s",
296 (int)child, shell_cmd);
297 return child;
298 }
299 }
300
301 static struct timeval *time_elapsed(struct timeval *result,
302 const struct timeval *start_time)
303 {
304 gettimeofday(result, NULL);
305 result->tv_sec -= start_time->tv_sec;
306 result->tv_usec -= start_time->tv_usec;
307 while (result->tv_usec < 0) {
308 result->tv_usec += 1000000L;
309 result->tv_sec--;
310 }
311 return result;
312 }
313
314 static int restart_kill(struct thread *t_kill)
315 {
316 struct restart_info *restart = THREAD_ARG(t_kill);
317 struct timeval delay;
318
319 time_elapsed(&delay, &restart->time);
320 zlog_warn(
321 "Warning: %s %s child process %d still running after "
322 "%ld seconds, sending signal %d",
323 restart->what, restart->name, (int)restart->pid,
324 (long)delay.tv_sec, (restart->kills ? SIGKILL : SIGTERM));
325 kill(-restart->pid, (restart->kills ? SIGKILL : SIGTERM));
326 restart->kills++;
327 restart->t_kill = NULL;
328 thread_add_timer(master, restart_kill, restart, gs.restart_timeout,
329 &restart->t_kill);
330 return 0;
331 }
332
333 static struct restart_info *find_child(pid_t child)
334 {
335 struct daemon *dmn;
336 if (gs.restart.pid == child)
337 return &gs.restart;
338
339 for (dmn = gs.daemons; dmn; dmn = dmn->next) {
340 if (dmn->restart.pid == child)
341 return &dmn->restart;
342 }
343 return NULL;
344 }
345
346 static void sigchild(void)
347 {
348 pid_t child;
349 int status;
350 const char *name;
351 const char *what;
352 struct restart_info *restart;
353 struct daemon *dmn;
354
355 switch (child = waitpid(-1, &status, WNOHANG)) {
356 case -1:
357 flog_err_sys(EC_LIB_SYSTEM_CALL, "waitpid failed: %s",
358 safe_strerror(errno));
359 return;
360 case 0:
361 zlog_warn("SIGCHLD received, but waitpid did not reap a child");
362 return;
363 }
364
365 if (child == integrated_write_pid) {
366 integrated_write_sigchld(status);
367 return;
368 }
369
370 if ((restart = find_child(child)) != NULL) {
371 name = restart->name;
372 what = restart->what;
373 restart->pid = 0;
374 gs.numpids--;
375 thread_cancel(restart->t_kill);
376 restart->t_kill = NULL;
377 /* Update restart time to reflect the time the command
378 * completed. */
379 gettimeofday(&restart->time, NULL);
380 } else {
381 flog_err_sys(
382 EC_LIB_SYSTEM_CALL,
383 "waitpid returned status for an unknown child process %d",
384 (int)child);
385 name = "(unknown)";
386 what = "background";
387 }
388 if (WIFSTOPPED(status))
389 zlog_warn("warning: %s %s process %d is stopped", what, name,
390 (int)child);
391 else if (WIFSIGNALED(status))
392 zlog_warn("%s %s process %d terminated due to signal %d", what,
393 name, (int)child, WTERMSIG(status));
394 else if (WIFEXITED(status)) {
395 if (WEXITSTATUS(status) != 0)
396 zlog_warn(
397 "%s %s process %d exited with non-zero status %d",
398 what, name, (int)child, WEXITSTATUS(status));
399 else {
400 zlog_debug("%s %s process %d exited normally", what,
401 name, (int)child);
402
403 if (restart && restart != &gs.restart) {
404 dmn = container_of(restart, struct daemon,
405 restart);
406 restart_done(dmn);
407 } else if (restart)
408 for (dmn = gs.daemons; dmn; dmn = dmn->next)
409 restart_done(dmn);
410 }
411 } else
412 flog_err_sys(
413 EC_LIB_SYSTEM_CALL,
414 "cannot interpret %s %s process %d wait status 0x%x",
415 what, name, (int)child, status);
416 phase_check();
417 }
418
419 static int run_job(struct restart_info *restart, const char *cmdtype,
420 const char *command, int force, int update_interval)
421 {
422 struct timeval delay;
423
424 if (gs.loglevel > LOG_DEBUG + 1)
425 zlog_debug("attempting to %s %s", cmdtype, restart->name);
426
427 if (restart->pid) {
428 if (gs.loglevel > LOG_DEBUG + 1)
429 zlog_debug(
430 "cannot %s %s, previous pid %d still running",
431 cmdtype, restart->name, (int)restart->pid);
432 return -1;
433 }
434
435 /* Note: time_elapsed test must come before the force test, since we
436 need
437 to make sure that delay is initialized for use below in updating the
438 restart interval. */
439 if ((time_elapsed(&delay, &restart->time)->tv_sec < restart->interval)
440 && !force) {
441 if (gs.loglevel > LOG_DEBUG + 1)
442 zlog_debug(
443 "postponing %s %s: "
444 "elapsed time %ld < retry interval %ld",
445 cmdtype, restart->name, (long)delay.tv_sec,
446 restart->interval);
447 return -1;
448 }
449
450 gettimeofday(&restart->time, NULL);
451 restart->kills = 0;
452 {
453 char cmd[strlen(command) + strlen(restart->name) + 1];
454 snprintf(cmd, sizeof(cmd), command, restart->name);
455 if ((restart->pid = run_background(cmd)) > 0) {
456 restart->t_kill = NULL;
457 thread_add_timer(master, restart_kill, restart,
458 gs.restart_timeout, &restart->t_kill);
459 restart->what = cmdtype;
460 gs.numpids++;
461 } else
462 restart->pid = 0;
463 }
464
465 /* Calculate the new restart interval. */
466 if (update_interval) {
467 if (delay.tv_sec > 2 * gs.max_restart_interval)
468 restart->interval = gs.min_restart_interval;
469 else if ((restart->interval *= 2) > gs.max_restart_interval)
470 restart->interval = gs.max_restart_interval;
471 if (gs.loglevel > LOG_DEBUG + 1)
472 zlog_debug("restart %s interval is now %ld",
473 restart->name, restart->interval);
474 }
475 return restart->pid;
476 }
477
478 #define SET_READ_HANDLER(DMN) \
479 do { \
480 (DMN)->t_read = NULL; \
481 thread_add_read(master, handle_read, (DMN), (DMN)->fd, \
482 &(DMN)->t_read); \
483 } while (0);
484
485 #define SET_WAKEUP_DOWN(DMN) \
486 do { \
487 (DMN)->t_wakeup = NULL; \
488 thread_add_timer_msec(master, wakeup_down, (DMN), \
489 FUZZY(gs.period), &(DMN)->t_wakeup); \
490 } while (0);
491
492 #define SET_WAKEUP_UNRESPONSIVE(DMN) \
493 do { \
494 (DMN)->t_wakeup = NULL; \
495 thread_add_timer_msec(master, wakeup_unresponsive, (DMN), \
496 FUZZY(gs.period), &(DMN)->t_wakeup); \
497 } while (0);
498
499 #define SET_WAKEUP_ECHO(DMN) \
500 do { \
501 (DMN)->t_wakeup = NULL; \
502 thread_add_timer_msec(master, wakeup_send_echo, (DMN), \
503 FUZZY(gs.period), &(DMN)->t_wakeup); \
504 } while (0);
505
506 static int wakeup_down(struct thread *t_wakeup)
507 {
508 struct daemon *dmn = THREAD_ARG(t_wakeup);
509
510 dmn->t_wakeup = NULL;
511 if (try_connect(dmn) < 0)
512 SET_WAKEUP_DOWN(dmn);
513 if ((dmn->connect_tries > 1) && (dmn->state != DAEMON_UP))
514 try_restart(dmn);
515 return 0;
516 }
517
518 static int wakeup_init(struct thread *t_wakeup)
519 {
520 struct daemon *dmn = THREAD_ARG(t_wakeup);
521
522 dmn->t_wakeup = NULL;
523 if (try_connect(dmn) < 0) {
524 flog_err(EC_WATCHFRR_CONNECTION,
525 "%s state -> down : initial connection attempt failed",
526 dmn->name);
527 dmn->state = DAEMON_DOWN;
528 }
529 phase_check();
530 return 0;
531 }
532
533 static void restart_done(struct daemon *dmn)
534 {
535 if (dmn->state != DAEMON_DOWN) {
536 zlog_warn("wtf?");
537 return;
538 }
539 if (dmn->t_wakeup)
540 THREAD_OFF(dmn->t_wakeup);
541 if (try_connect(dmn) < 0)
542 SET_WAKEUP_DOWN(dmn);
543 }
544
545 static void daemon_down(struct daemon *dmn, const char *why)
546 {
547 if (IS_UP(dmn) || (dmn->state == DAEMON_INIT))
548 flog_err(EC_WATCHFRR_CONNECTION, "%s state -> down : %s",
549 dmn->name, why);
550 else if (gs.loglevel > LOG_DEBUG)
551 zlog_debug("%s still down : %s", dmn->name, why);
552 if (IS_UP(dmn))
553 gs.numdown++;
554 dmn->state = DAEMON_DOWN;
555 if (dmn->fd >= 0) {
556 close(dmn->fd);
557 dmn->fd = -1;
558 }
559 THREAD_OFF(dmn->t_read);
560 THREAD_OFF(dmn->t_write);
561 THREAD_OFF(dmn->t_wakeup);
562 if (try_connect(dmn) < 0)
563 SET_WAKEUP_DOWN(dmn);
564 phase_check();
565 }
566
567 static int handle_read(struct thread *t_read)
568 {
569 struct daemon *dmn = THREAD_ARG(t_read);
570 static const char resp[sizeof(PING_TOKEN) + 4] = PING_TOKEN "\n";
571 char buf[sizeof(resp) + 100];
572 ssize_t rc;
573 struct timeval delay;
574
575 dmn->t_read = NULL;
576 if ((rc = read(dmn->fd, buf, sizeof(buf))) < 0) {
577 char why[100];
578
579 if (ERRNO_IO_RETRY(errno)) {
580 /* Pretend it never happened. */
581 SET_READ_HANDLER(dmn);
582 return 0;
583 }
584 snprintf(why, sizeof(why), "unexpected read error: %s",
585 safe_strerror(errno));
586 daemon_down(dmn, why);
587 return 0;
588 }
589 if (rc == 0) {
590 daemon_down(dmn, "read returned EOF");
591 return 0;
592 }
593 if (!dmn->echo_sent.tv_sec) {
594 char why[sizeof(buf) + 100];
595 snprintf(why, sizeof(why),
596 "unexpected read returns %d bytes: %.*s", (int)rc,
597 (int)rc, buf);
598 daemon_down(dmn, why);
599 return 0;
600 }
601
602 /* We are expecting an echo response: is there any chance that the
603 response would not be returned entirely in the first read? That
604 seems inconceivable... */
605 if ((rc != sizeof(resp)) || memcmp(buf, resp, sizeof(resp))) {
606 char why[100 + sizeof(buf)];
607 snprintf(why, sizeof(why),
608 "read returned bad echo response of %d bytes "
609 "(expecting %u): %.*s",
610 (int)rc, (unsigned int)sizeof(resp), (int)rc, buf);
611 daemon_down(dmn, why);
612 return 0;
613 }
614
615 time_elapsed(&delay, &dmn->echo_sent);
616 dmn->echo_sent.tv_sec = 0;
617 if (dmn->state == DAEMON_UNRESPONSIVE) {
618 if (delay.tv_sec < gs.timeout) {
619 dmn->state = DAEMON_UP;
620 zlog_warn(
621 "%s state -> up : echo response received after %ld.%06ld "
622 "seconds",
623 dmn->name, (long)delay.tv_sec,
624 (long)delay.tv_usec);
625 } else
626 zlog_warn(
627 "%s: slow echo response finally received after %ld.%06ld "
628 "seconds",
629 dmn->name, (long)delay.tv_sec,
630 (long)delay.tv_usec);
631 } else if (gs.loglevel > LOG_DEBUG + 1)
632 zlog_debug("%s: echo response received after %ld.%06ld seconds",
633 dmn->name, (long)delay.tv_sec, (long)delay.tv_usec);
634
635 SET_READ_HANDLER(dmn);
636 if (dmn->t_wakeup)
637 thread_cancel(dmn->t_wakeup);
638 SET_WAKEUP_ECHO(dmn);
639
640 return 0;
641 }
642
643 /*
644 * Wait till we notice that all daemons are ready before
645 * we send we are ready to systemd
646 */
647 static void daemon_send_ready(int exitcode)
648 {
649 FILE *fp;
650 static int sent = 0;
651
652 if (sent)
653 return;
654
655 if (exitcode == 0)
656 zlog_notice("all daemons up, doing startup-complete notify");
657 else if (gs.numdown < gs.numdaemons)
658 flog_err(EC_WATCHFRR_CONNECTION,
659 "startup did not complete within timeout"
660 " (%d/%d daemons running)",
661 gs.numdaemons - gs.numdown, gs.numdaemons);
662 else {
663 flog_err(EC_WATCHFRR_CONNECTION,
664 "all configured daemons failed to start"
665 " -- exiting watchfrr");
666 exit(exitcode);
667
668 }
669
670 frr_detach();
671
672 fp = fopen(DAEMON_VTY_DIR "/watchfrr.started", "w");
673 if (fp)
674 fclose(fp);
675 #if defined HAVE_SYSTEMD
676 systemd_send_started(master, 0);
677 #endif
678 sent = 1;
679 }
680
681 static void daemon_up(struct daemon *dmn, const char *why)
682 {
683 dmn->state = DAEMON_UP;
684 gs.numdown--;
685 dmn->connect_tries = 0;
686 zlog_notice("%s state -> up : %s", dmn->name, why);
687 if (gs.numdown == 0)
688 daemon_send_ready(0);
689 SET_WAKEUP_ECHO(dmn);
690 phase_check();
691 }
692
693 static int check_connect(struct thread *t_write)
694 {
695 struct daemon *dmn = THREAD_ARG(t_write);
696 int sockerr;
697 socklen_t reslen = sizeof(sockerr);
698
699 dmn->t_write = NULL;
700 if (getsockopt(dmn->fd, SOL_SOCKET, SO_ERROR, (char *)&sockerr, &reslen)
701 < 0) {
702 zlog_warn("%s: check_connect: getsockopt failed: %s", dmn->name,
703 safe_strerror(errno));
704 daemon_down(dmn,
705 "getsockopt failed checking connection success");
706 return 0;
707 }
708 if ((reslen == sizeof(sockerr)) && sockerr) {
709 char why[100];
710 snprintf(
711 why, sizeof(why),
712 "getsockopt reports that connection attempt failed: %s",
713 safe_strerror(sockerr));
714 daemon_down(dmn, why);
715 return 0;
716 }
717
718 daemon_up(dmn, "delayed connect succeeded");
719 return 0;
720 }
721
722 static int wakeup_connect_hanging(struct thread *t_wakeup)
723 {
724 struct daemon *dmn = THREAD_ARG(t_wakeup);
725 char why[100];
726
727 dmn->t_wakeup = NULL;
728 snprintf(why, sizeof(why),
729 "connection attempt timed out after %ld seconds", gs.timeout);
730 daemon_down(dmn, why);
731 return 0;
732 }
733
734 /* Making connection to protocol daemon. */
735 static int try_connect(struct daemon *dmn)
736 {
737 int sock;
738 struct sockaddr_un addr;
739 socklen_t len;
740
741 if (gs.loglevel > LOG_DEBUG + 1)
742 zlog_debug("%s: attempting to connect", dmn->name);
743 dmn->connect_tries++;
744
745 memset(&addr, 0, sizeof(struct sockaddr_un));
746 addr.sun_family = AF_UNIX;
747 snprintf(addr.sun_path, sizeof(addr.sun_path), "%s/%s.vty", gs.vtydir,
748 dmn->name);
749 #ifdef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN
750 len = addr.sun_len = SUN_LEN(&addr);
751 #else
752 len = sizeof(addr.sun_family) + strlen(addr.sun_path);
753 #endif /* HAVE_STRUCT_SOCKADDR_UN_SUN_LEN */
754
755 /* Quick check to see if we might succeed before we go to the trouble
756 of creating a socket. */
757 if (access(addr.sun_path, W_OK) < 0) {
758 if (errno != ENOENT)
759 flog_err_sys(EC_LIB_SYSTEM_CALL,
760 "%s: access to socket %s denied: %s",
761 dmn->name, addr.sun_path,
762 safe_strerror(errno));
763 return -1;
764 }
765
766 if ((sock = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
767 flog_err_sys(EC_LIB_SOCKET, "%s(%s): cannot make socket: %s",
768 __func__, addr.sun_path, safe_strerror(errno));
769 return -1;
770 }
771
772 if (set_nonblocking(sock) < 0 || set_cloexec(sock) < 0) {
773 flog_err_sys(EC_LIB_SYSTEM_CALL,
774 "%s(%s): set_nonblocking/cloexec(%d) failed",
775 __func__, addr.sun_path, sock);
776 close(sock);
777 return -1;
778 }
779
780 if (connect(sock, (struct sockaddr *)&addr, len) < 0) {
781 if ((errno != EINPROGRESS) && (errno != EWOULDBLOCK)) {
782 if (gs.loglevel > LOG_DEBUG)
783 zlog_debug("%s(%s): connect failed: %s",
784 __func__, addr.sun_path,
785 safe_strerror(errno));
786 close(sock);
787 return -1;
788 }
789 if (gs.loglevel > LOG_DEBUG)
790 zlog_debug("%s: connection in progress", dmn->name);
791 dmn->state = DAEMON_CONNECTING;
792 dmn->fd = sock;
793 dmn->t_write = NULL;
794 thread_add_write(master, check_connect, dmn, dmn->fd,
795 &dmn->t_write);
796 dmn->t_wakeup = NULL;
797 thread_add_timer(master, wakeup_connect_hanging, dmn,
798 gs.timeout, &dmn->t_wakeup);
799 SET_READ_HANDLER(dmn);
800 return 0;
801 }
802
803 dmn->fd = sock;
804 SET_READ_HANDLER(dmn);
805 daemon_up(dmn, "connect succeeded");
806 return 1;
807 }
808
809 static int phase_hanging(struct thread *t_hanging)
810 {
811 gs.t_phase_hanging = NULL;
812 flog_err(EC_WATCHFRR_CONNECTION,
813 "Phase [%s] hanging for %ld seconds, aborting phased restart",
814 phase_str[gs.phase], PHASE_TIMEOUT);
815 gs.phase = PHASE_NONE;
816 return 0;
817 }
818
819 static void set_phase(restart_phase_t new_phase)
820 {
821 gs.phase = new_phase;
822 if (gs.t_phase_hanging)
823 thread_cancel(gs.t_phase_hanging);
824 gs.t_phase_hanging = NULL;
825 thread_add_timer(master, phase_hanging, NULL, PHASE_TIMEOUT,
826 &gs.t_phase_hanging);
827 }
828
829 static void phase_check(void)
830 {
831 struct daemon *dmn;
832
833 switch (gs.phase) {
834 case PHASE_NONE:
835 break;
836
837 case PHASE_INIT:
838 for (dmn = gs.daemons; dmn; dmn = dmn->next)
839 if (dmn->state == DAEMON_INIT)
840 return;
841
842 /* startup complete, everything out of INIT */
843 gs.phase = PHASE_NONE;
844 for (dmn = gs.daemons; dmn; dmn = dmn->next)
845 if (dmn->state == DAEMON_DOWN) {
846 SET_WAKEUP_DOWN(dmn);
847 try_restart(dmn);
848 }
849 break;
850 case PHASE_STOPS_PENDING:
851 if (gs.numpids)
852 break;
853 zlog_info(
854 "Phased restart: all routing daemon stop jobs have completed.");
855 set_phase(PHASE_WAITING_DOWN);
856
857 /*FALLTHRU*/
858 case PHASE_WAITING_DOWN:
859 if (gs.numdown + IS_UP(gs.special) < gs.numdaemons)
860 break;
861 zlog_info("Phased restart: all routing daemons now down.");
862 run_job(&gs.special->restart, "restart", gs.restart_command, 1,
863 1);
864 set_phase(PHASE_ZEBRA_RESTART_PENDING);
865
866 /*FALLTHRU*/
867 case PHASE_ZEBRA_RESTART_PENDING:
868 if (gs.special->restart.pid)
869 break;
870 zlog_info("Phased restart: %s restart job completed.",
871 gs.special->name);
872 set_phase(PHASE_WAITING_ZEBRA_UP);
873
874 /*FALLTHRU*/
875 case PHASE_WAITING_ZEBRA_UP:
876 if (!IS_UP(gs.special))
877 break;
878 zlog_info("Phased restart: %s is now up.", gs.special->name);
879 {
880 struct daemon *dmn;
881 for (dmn = gs.daemons; dmn; dmn = dmn->next) {
882 if (dmn != gs.special)
883 run_job(&dmn->restart, "start",
884 gs.start_command, 1, 0);
885 }
886 }
887 gs.phase = PHASE_NONE;
888 THREAD_OFF(gs.t_phase_hanging);
889 zlog_notice("Phased global restart has completed.");
890 break;
891 }
892 }
893
894 static void try_restart(struct daemon *dmn)
895 {
896 if (watch_only)
897 return;
898
899 if (dmn != gs.special) {
900 if ((gs.special->state == DAEMON_UP)
901 && (gs.phase == PHASE_NONE))
902 run_job(&dmn->restart, "restart", gs.restart_command, 0,
903 1);
904 else
905 zlog_debug(
906 "%s: postponing restart attempt because master %s daemon "
907 "not up [%s], or phased restart in progress",
908 dmn->name, gs.special->name,
909 state_str[gs.special->state]);
910 return;
911 }
912
913 if ((gs.phase != PHASE_NONE) || gs.numpids) {
914 if (gs.loglevel > LOG_DEBUG + 1)
915 zlog_debug(
916 "postponing phased global restart: restart already in "
917 "progress [%s], or outstanding child processes [%d]",
918 phase_str[gs.phase], gs.numpids);
919 return;
920 }
921 /* Is it too soon for a restart? */
922 {
923 struct timeval delay;
924 if (time_elapsed(&delay, &gs.special->restart.time)->tv_sec
925 < gs.special->restart.interval) {
926 if (gs.loglevel > LOG_DEBUG + 1)
927 zlog_debug(
928 "postponing phased global restart: "
929 "elapsed time %ld < retry interval %ld",
930 (long)delay.tv_sec,
931 gs.special->restart.interval);
932 return;
933 }
934 }
935 run_job(&gs.restart, "restart", gs.restart_command, 0, 1);
936 }
937
938 static int wakeup_unresponsive(struct thread *t_wakeup)
939 {
940 struct daemon *dmn = THREAD_ARG(t_wakeup);
941
942 dmn->t_wakeup = NULL;
943 if (dmn->state != DAEMON_UNRESPONSIVE)
944 flog_err(EC_WATCHFRR_CONNECTION,
945 "%s: no longer unresponsive (now %s), "
946 "wakeup should have been cancelled!",
947 dmn->name, state_str[dmn->state]);
948 else {
949 SET_WAKEUP_UNRESPONSIVE(dmn);
950 try_restart(dmn);
951 }
952 return 0;
953 }
954
955 static int wakeup_no_answer(struct thread *t_wakeup)
956 {
957 struct daemon *dmn = THREAD_ARG(t_wakeup);
958
959 dmn->t_wakeup = NULL;
960 dmn->state = DAEMON_UNRESPONSIVE;
961 flog_err(EC_WATCHFRR_CONNECTION,
962 "%s state -> unresponsive : no response yet to ping "
963 "sent %ld seconds ago",
964 dmn->name, gs.timeout);
965 SET_WAKEUP_UNRESPONSIVE(dmn);
966 try_restart(dmn);
967 return 0;
968 }
969
970 static int wakeup_send_echo(struct thread *t_wakeup)
971 {
972 static const char echocmd[] = "echo " PING_TOKEN;
973 ssize_t rc;
974 struct daemon *dmn = THREAD_ARG(t_wakeup);
975
976 dmn->t_wakeup = NULL;
977 if (((rc = write(dmn->fd, echocmd, sizeof(echocmd))) < 0)
978 || ((size_t)rc != sizeof(echocmd))) {
979 char why[100 + sizeof(echocmd)];
980 snprintf(why, sizeof(why),
981 "write '%s' returned %d instead of %u", echocmd,
982 (int)rc, (unsigned int)sizeof(echocmd));
983 daemon_down(dmn, why);
984 } else {
985 gettimeofday(&dmn->echo_sent, NULL);
986 dmn->t_wakeup = NULL;
987 thread_add_timer(master, wakeup_no_answer, dmn, gs.timeout,
988 &dmn->t_wakeup);
989 }
990 return 0;
991 }
992
993 bool check_all_up(void)
994 {
995 struct daemon *dmn;
996
997 for (dmn = gs.daemons; dmn; dmn = dmn->next)
998 if (dmn->state != DAEMON_UP)
999 return false;
1000 return true;
1001 }
1002
1003 void watchfrr_status(struct vty *vty)
1004 {
1005 struct daemon *dmn;
1006 struct timeval delay;
1007
1008 vty_out(vty, "watchfrr global phase: %s\n", phase_str[gs.phase]);
1009 if (gs.restart.pid)
1010 vty_out(vty, " global restart running, pid %ld\n",
1011 (long)gs.restart.pid);
1012
1013 for (dmn = gs.daemons; dmn; dmn = dmn->next) {
1014 vty_out(vty, " %-20s %s\n", dmn->name, state_str[dmn->state]);
1015 if (dmn->restart.pid)
1016 vty_out(vty, " restart running, pid %ld\n",
1017 (long)dmn->restart.pid);
1018 else if (dmn->state == DAEMON_DOWN &&
1019 time_elapsed(&delay, &dmn->restart.time)->tv_sec
1020 < dmn->restart.interval)
1021 vty_out(vty, " restarting in %ld seconds"
1022 " (%lds backoff interval)\n",
1023 dmn->restart.interval - delay.tv_sec,
1024 dmn->restart.interval);
1025 }
1026 }
1027
1028 static void sigint(void)
1029 {
1030 zlog_notice("Terminating on signal");
1031 systemd_send_stopping();
1032 exit(0);
1033 }
1034
1035 static int valid_command(const char *cmd)
1036 {
1037 char *p;
1038
1039 return ((p = strchr(cmd, '%')) != NULL) && (*(p + 1) == 's')
1040 && !strchr(p + 1, '%');
1041 }
1042
1043 /* This is an ugly hack to circumvent problems with passing command-line
1044 arguments that contain spaces. The fix is to use a configuration file. */
1045 static char *translate_blanks(const char *cmd, const char *blankstr)
1046 {
1047 char *res;
1048 char *p;
1049 size_t bslen = strlen(blankstr);
1050
1051 if (!(res = strdup(cmd))) {
1052 perror("strdup");
1053 exit(1);
1054 }
1055 while ((p = strstr(res, blankstr)) != NULL) {
1056 *p = ' ';
1057 if (bslen != 1)
1058 memmove(p + 1, p + bslen, strlen(p + bslen) + 1);
1059 }
1060 return res;
1061 }
1062
1063 static int startup_timeout(struct thread *t_wakeup)
1064 {
1065 daemon_send_ready(1);
1066 return 0;
1067 }
1068
1069 static void watchfrr_init(int argc, char **argv)
1070 {
1071 const char *special = "zebra";
1072 int i;
1073 struct daemon *dmn, **add = &gs.daemons;
1074 char alldaemons[512] = "", *p = alldaemons;
1075
1076 thread_add_timer_msec(master, startup_timeout, NULL, STARTUP_TIMEOUT,
1077 &gs.t_startup_timeout);
1078
1079 for (i = optind; i < argc; i++) {
1080 dmn = XCALLOC(MTYPE_WATCHFRR_DAEMON, sizeof(*dmn));
1081
1082 dmn->name = dmn->restart.name = argv[i];
1083 dmn->state = DAEMON_INIT;
1084 gs.numdaemons++;
1085 gs.numdown++;
1086 dmn->fd = -1;
1087 dmn->t_wakeup = NULL;
1088 thread_add_timer_msec(master, wakeup_init, dmn, 0,
1089 &dmn->t_wakeup);
1090 dmn->restart.interval = gs.min_restart_interval;
1091 *add = dmn;
1092 add = &dmn->next;
1093
1094 if (!strcmp(dmn->name, special))
1095 gs.special = dmn;
1096 }
1097
1098 if (!gs.daemons) {
1099 fprintf(stderr,
1100 "Must specify one or more daemons to monitor.\n\n");
1101 frr_help_exit(1);
1102 }
1103 if (!watch_only && !gs.special) {
1104 fprintf(stderr, "\"%s\" daemon must be in daemon lists\n\n",
1105 special);
1106 frr_help_exit(1);
1107 }
1108
1109 for (dmn = gs.daemons; dmn; dmn = dmn->next) {
1110 snprintf(p, alldaemons + sizeof(alldaemons) - p, "%s%s",
1111 (p == alldaemons) ? "" : " ", dmn->name);
1112 p += strlen(p);
1113 }
1114 zlog_notice("%s %s watching [%s]%s", progname, FRR_VERSION, alldaemons,
1115 watch_only ? ", monitor mode" : "");
1116 }
1117
1118 struct zebra_privs_t watchfrr_privs = {
1119 #ifdef VTY_GROUP
1120 .vty_group = VTY_GROUP,
1121 #endif
1122 };
1123
1124 static struct quagga_signal_t watchfrr_signals[] = {
1125 {
1126 .signal = SIGINT,
1127 .handler = sigint,
1128 },
1129 {
1130 .signal = SIGTERM,
1131 .handler = sigint,
1132 },
1133 {
1134 .signal = SIGCHLD,
1135 .handler = sigchild,
1136 },
1137 };
1138
1139 FRR_DAEMON_INFO(watchfrr, WATCHFRR,
1140 .flags = FRR_NO_PRIVSEP | FRR_NO_TCPVTY | FRR_LIMITED_CLI
1141 | FRR_NO_CFG_PID_DRY | FRR_NO_ZCLIENT
1142 | FRR_DETACH_LATER,
1143
1144 .printhelp = printhelp,
1145 .copyright = "Copyright 2004 Andrew J. Schorr",
1146
1147 .signals = watchfrr_signals,
1148 .n_signals = array_size(watchfrr_signals),
1149
1150 .privs = &watchfrr_privs, )
1151
1152 #define DEPRECATED_OPTIONS "aAezR:"
1153
1154 int main(int argc, char **argv)
1155 {
1156 int opt;
1157 const char *blankstr = NULL;
1158
1159 frr_preinit(&watchfrr_di, argc, argv);
1160 progname = watchfrr_di.progname;
1161
1162 frr_opt_add("b:dk:l:i:p:r:S:s:t:T:" DEPRECATED_OPTIONS, longopts, "");
1163
1164 gs.restart.name = "all";
1165 while ((opt = frr_getopt(argc, argv, NULL)) != EOF) {
1166 if (opt && opt < 128 && strchr(DEPRECATED_OPTIONS, opt)) {
1167 fprintf(stderr,
1168 "The -%c option no longer exists.\n"
1169 "Please refer to the watchfrr(8) man page.\n",
1170 opt);
1171 exit(1);
1172 }
1173
1174 switch (opt) {
1175 case 0:
1176 break;
1177 case 'b':
1178 blankstr = optarg;
1179 break;
1180 case OPTION_DRY:
1181 watch_only = true;
1182 break;
1183 case 'k':
1184 if (!valid_command(optarg)) {
1185 fprintf(stderr,
1186 "Invalid kill command, must contain '%%s': %s\n",
1187 optarg);
1188 frr_help_exit(1);
1189 }
1190 gs.stop_command = optarg;
1191 break;
1192 case 'l': {
1193 char garbage[3];
1194 if ((sscanf(optarg, "%d%1s", &gs.loglevel, garbage)
1195 != 1)
1196 || (gs.loglevel < LOG_EMERG)) {
1197 fprintf(stderr,
1198 "Invalid loglevel argument: %s\n",
1199 optarg);
1200 frr_help_exit(1);
1201 }
1202 } break;
1203 case OPTION_MINRESTART: {
1204 char garbage[3];
1205 if ((sscanf(optarg, "%ld%1s", &gs.min_restart_interval,
1206 garbage)
1207 != 1)
1208 || (gs.min_restart_interval < 0)) {
1209 fprintf(stderr,
1210 "Invalid min_restart_interval argument: %s\n",
1211 optarg);
1212 frr_help_exit(1);
1213 }
1214 } break;
1215 case OPTION_MAXRESTART: {
1216 char garbage[3];
1217 if ((sscanf(optarg, "%ld%1s", &gs.max_restart_interval,
1218 garbage)
1219 != 1)
1220 || (gs.max_restart_interval < 0)) {
1221 fprintf(stderr,
1222 "Invalid max_restart_interval argument: %s\n",
1223 optarg);
1224 frr_help_exit(1);
1225 }
1226 } break;
1227 case 'i': {
1228 char garbage[3];
1229 int period;
1230 if ((sscanf(optarg, "%d%1s", &period, garbage) != 1)
1231 || (gs.period < 1)) {
1232 fprintf(stderr,
1233 "Invalid interval argument: %s\n",
1234 optarg);
1235 frr_help_exit(1);
1236 }
1237 gs.period = 1000 * period;
1238 } break;
1239 case 'p':
1240 watchfrr_di.pid_file = optarg;
1241 break;
1242 case 'r':
1243 if (!valid_command(optarg)) {
1244 fprintf(stderr,
1245 "Invalid restart command, must contain '%%s': %s\n",
1246 optarg);
1247 frr_help_exit(1);
1248 }
1249 gs.restart_command = optarg;
1250 break;
1251 case 's':
1252 if (!valid_command(optarg)) {
1253 fprintf(stderr,
1254 "Invalid start command, must contain '%%s': %s\n",
1255 optarg);
1256 frr_help_exit(1);
1257 }
1258 gs.start_command = optarg;
1259 break;
1260 case 'S':
1261 gs.vtydir = optarg;
1262 break;
1263 case 't': {
1264 char garbage[3];
1265 if ((sscanf(optarg, "%ld%1s", &gs.timeout, garbage)
1266 != 1)
1267 || (gs.timeout < 1)) {
1268 fprintf(stderr,
1269 "Invalid timeout argument: %s\n",
1270 optarg);
1271 frr_help_exit(1);
1272 }
1273 } break;
1274 case 'T': {
1275 char garbage[3];
1276 if ((sscanf(optarg, "%ld%1s", &gs.restart_timeout,
1277 garbage)
1278 != 1)
1279 || (gs.restart_timeout < 1)) {
1280 fprintf(stderr,
1281 "Invalid restart timeout argument: %s\n",
1282 optarg);
1283 frr_help_exit(1);
1284 }
1285 } break;
1286 default:
1287 fputs("Invalid option.\n", stderr);
1288 frr_help_exit(1);
1289 }
1290 }
1291
1292 if (watch_only
1293 && (gs.start_command || gs.stop_command || gs.restart_command)) {
1294 fputs("Options -r/-s/-k are not used when --dry is active.\n",
1295 stderr);
1296 }
1297 if (!watch_only
1298 && (!gs.restart_command || !gs.start_command || !gs.stop_command)) {
1299 fprintf(stderr,
1300 "Options -s (start), -k (kill), and -r (restart) are required.\n");
1301 frr_help_exit(1);
1302 }
1303
1304 if (blankstr) {
1305 if (gs.restart_command)
1306 gs.restart_command =
1307 translate_blanks(gs.restart_command, blankstr);
1308 if (gs.start_command)
1309 gs.start_command =
1310 translate_blanks(gs.start_command, blankstr);
1311 if (gs.stop_command)
1312 gs.stop_command =
1313 translate_blanks(gs.stop_command, blankstr);
1314 }
1315
1316 gs.restart.interval = gs.min_restart_interval;
1317
1318 master = frr_init();
1319 watchfrr_error_init();
1320 watchfrr_init(argc, argv);
1321 watchfrr_vty_init();
1322
1323 frr_config_fork();
1324
1325 zlog_set_level(ZLOG_DEST_MONITOR, ZLOG_DISABLED);
1326 if (watchfrr_di.daemon_mode)
1327 zlog_set_level(ZLOG_DEST_SYSLOG, MIN(gs.loglevel, LOG_DEBUG));
1328 else
1329 zlog_set_level(ZLOG_DEST_STDOUT, MIN(gs.loglevel, LOG_DEBUG));
1330
1331 frr_run(master);
1332
1333 systemd_send_stopping();
1334 /* Not reached. */
1335 return 0;
1336 }
FRRouting mirror