2 * Copyright (c) 2008, 2009, 2010, 2012, 2013 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 #include "reconnect.h"
22 #include "poll-loop.h"
23 #include "openvswitch/vlog.h"
25 VLOG_DEFINE_THIS_MODULE(reconnect
);
29 STATE(BACKOFF, 1 << 1) \
30 STATE(CONNECTING, 1 << 3) \
31 STATE(ACTIVE, 1 << 4) \
33 STATE(RECONNECT, 1 << 6) \
34 STATE(LISTENING, 1 << 7)
36 #define STATE(NAME, VALUE) S_##NAME = VALUE,
42 is_connected_state(enum state state
)
44 return (state
& (S_ACTIVE
| S_IDLE
)) != 0;
54 enum vlog_level info
; /* Used for informational messages. */
58 long long int state_entered
;
60 long long int last_activity
;
61 long long int last_connected
;
62 long long int last_disconnected
;
63 unsigned int max_tries
;
65 /* These values are simply for statistics reporting, not otherwise used
66 * directly by anything internal. */
67 long long int creation_time
;
68 unsigned int n_attempted_connections
, n_successful_connections
;
69 unsigned int total_connected_duration
;
73 static void reconnect_transition__(struct reconnect
*, long long int now
,
75 static long long int reconnect_deadline__(const struct reconnect
*);
76 static bool reconnect_may_retry(struct reconnect
*);
79 reconnect_state_name__(enum state state
)
82 #define STATE(NAME, VALUE) case S_##NAME: return #NAME;
89 /* Creates and returns a new reconnect FSM with default settings. The FSM is
90 * initially disabled. The caller will likely want to call reconnect_enable()
91 * and reconnect_set_name() on the returned object. */
93 reconnect_create(long long int now
)
95 struct reconnect
*fsm
= xzalloc(sizeof *fsm
);
97 fsm
->name
= xstrdup("void");
98 fsm
->min_backoff
= RECONNECT_DEFAULT_MIN_BACKOFF
;
99 fsm
->max_backoff
= RECONNECT_DEFAULT_MAX_BACKOFF
;
100 fsm
->probe_interval
= RECONNECT_DEFAULT_PROBE_INTERVAL
;
101 fsm
->passive
= false;
102 fsm
->info
= VLL_INFO
;
105 fsm
->state_entered
= now
;
107 fsm
->last_activity
= now
;
108 fsm
->last_connected
= LLONG_MAX
;
109 fsm
->last_disconnected
= LLONG_MAX
;
110 fsm
->max_tries
= UINT_MAX
;
111 fsm
->creation_time
= now
;
118 reconnect_destroy(struct reconnect
*fsm
)
126 /* If 'quiet' is true, 'fsm' will log informational messages at level VLL_DBG,
127 * by default keeping them out of log files. This is appropriate if the
128 * connection is one that is expected to be short-lived, so that the log
129 * messages are merely distracting.
131 * If 'quiet' is false, 'fsm' logs informational messages at level VLL_INFO.
132 * This is the default.
134 * This setting has no effect on the log level of debugging, warning, or error
137 reconnect_set_quiet(struct reconnect
*fsm
, bool quiet
)
139 fsm
->info
= quiet
? VLL_DBG
: VLL_INFO
;
142 /* Returns 'fsm''s name. */
144 reconnect_get_name(const struct reconnect
*fsm
)
149 /* Sets 'fsm''s name to 'name'. If 'name' is null, then "void" is used
152 * The name set for 'fsm' is used in log messages. */
154 reconnect_set_name(struct reconnect
*fsm
, const char *name
)
157 fsm
->name
= xstrdup(name
? name
: "void");
160 /* Return the minimum number of milliseconds to back off between consecutive
161 * connection attempts. The default is RECONNECT_DEFAULT_MIN_BACKOFF. */
163 reconnect_get_min_backoff(const struct reconnect
*fsm
)
165 return fsm
->min_backoff
;
168 /* Return the maximum number of milliseconds to back off between consecutive
169 * connection attempts. The default is RECONNECT_DEFAULT_MAX_BACKOFF. */
171 reconnect_get_max_backoff(const struct reconnect
*fsm
)
173 return fsm
->max_backoff
;
176 /* Returns the "probe interval" for 'fsm' in milliseconds. If this is zero, it
177 * disables the connection keepalive feature. If it is nonzero, then if the
178 * interval passes while 'fsm' is connected and without reconnect_activity()
179 * being called for 'fsm', reconnect_run() returns RECONNECT_PROBE. If the
180 * interval passes again without reconnect_activity() being called,
181 * reconnect_run() returns RECONNECT_DISCONNECT for 'fsm'. */
183 reconnect_get_probe_interval(const struct reconnect
*fsm
)
185 return fsm
->probe_interval
;
188 /* Limits the maximum number of times that 'fsm' will ask the client to try to
189 * reconnect to 'max_tries'. UINT_MAX (the default) means an unlimited number
192 * After the number of tries has expired, the 'fsm' will disable itself
193 * instead of backing off and retrying. */
195 reconnect_set_max_tries(struct reconnect
*fsm
, unsigned int max_tries
)
197 fsm
->max_tries
= max_tries
;
200 /* Returns the current remaining number of connection attempts, UINT_MAX if
201 * the number is unlimited. */
203 reconnect_get_max_tries(struct reconnect
*fsm
)
205 return fsm
->max_tries
;
208 /* Configures the backoff parameters for 'fsm'. 'min_backoff' is the minimum
209 * number of milliseconds, and 'max_backoff' is the maximum, between connection
210 * attempts. The current backoff is also the duration that 'fsm' is willing to
211 * wait for a given connection to succeed or fail.
213 * 'min_backoff' must be at least 1000, and 'max_backoff' must be greater than
214 * or equal to 'min_backoff'.
216 * Pass 0 for 'min_backoff' or 'max_backoff' or both to use the defaults. */
218 reconnect_set_backoff(struct reconnect
*fsm
, int min_backoff
, int max_backoff
)
220 fsm
->min_backoff
= MAX(min_backoff
, 1000);
221 fsm
->max_backoff
= (max_backoff
222 ? MAX(max_backoff
, 1000)
223 : RECONNECT_DEFAULT_MAX_BACKOFF
);
224 if (fsm
->min_backoff
> fsm
->max_backoff
) {
225 fsm
->max_backoff
= fsm
->min_backoff
;
228 if (fsm
->state
== S_BACKOFF
&& fsm
->backoff
> max_backoff
) {
229 fsm
->backoff
= max_backoff
;
233 /* Sets the "probe interval" for 'fsm' to 'probe_interval', in milliseconds.
234 * If this is zero, it disables the connection keepalive feature. If it is
235 * nonzero, then if the interval passes while 'fsm' is connected and without
236 * reconnect_activity() being called for 'fsm', reconnect_run() returns
237 * RECONNECT_PROBE. If the interval passes again without reconnect_activity()
238 * being called, reconnect_run() returns RECONNECT_DISCONNECT for 'fsm'.
240 * If 'probe_interval' is nonzero, then it will be forced to a value of at
243 reconnect_set_probe_interval(struct reconnect
*fsm
, int probe_interval
)
245 fsm
->probe_interval
= probe_interval
? MAX(1000, probe_interval
) : 0;
248 /* Returns true if 'fsm' is in passive mode, false if 'fsm' is in active mode
251 reconnect_is_passive(const struct reconnect
*fsm
)
256 /* Configures 'fsm' for active or passive mode. In active mode (the default),
257 * the FSM is attempting to connect to a remote host. In passive mode, the FSM
258 * is listening for connections from a remote host. */
260 reconnect_set_passive(struct reconnect
*fsm
, bool passive
, long long int now
)
262 if (fsm
->passive
!= passive
) {
263 fsm
->passive
= passive
;
266 ? fsm
->state
& (S_CONNECTING
| S_RECONNECT
)
267 : fsm
->state
== S_LISTENING
&& reconnect_may_retry(fsm
)) {
268 reconnect_transition__(fsm
, now
, S_BACKOFF
);
274 /* Returns true if 'fsm' has been enabled with reconnect_enable(). Calling
275 * another function that indicates a change in connection state, such as
276 * reconnect_disconnected() or reconnect_force_reconnect(), will also enable
277 * a reconnect FSM. */
279 reconnect_is_enabled(const struct reconnect
*fsm
)
281 return fsm
->state
!= S_VOID
;
284 /* If 'fsm' is disabled (the default for newly created FSMs), enables it, so
285 * that the next call to reconnect_run() for 'fsm' will return
288 * If 'fsm' is not disabled, this function has no effect. */
290 reconnect_enable(struct reconnect
*fsm
, long long int now
)
292 if (fsm
->state
== S_VOID
&& reconnect_may_retry(fsm
)) {
293 reconnect_transition__(fsm
, now
, S_BACKOFF
);
298 /* Disables 'fsm'. Until 'fsm' is enabled again, reconnect_run() will always
301 reconnect_disable(struct reconnect
*fsm
, long long int now
)
303 if (fsm
->state
!= S_VOID
) {
304 reconnect_transition__(fsm
, now
, S_VOID
);
308 /* If 'fsm' is enabled and currently connected (or attempting to connect),
309 * forces reconnect_run() for 'fsm' to return RECONNECT_DISCONNECT the next
310 * time it is called, which should cause the client to drop the connection (or
311 * attempt), back off, and then reconnect. */
313 reconnect_force_reconnect(struct reconnect
*fsm
, long long int now
)
315 if (fsm
->state
& (S_CONNECTING
| S_ACTIVE
| S_IDLE
)) {
316 reconnect_transition__(fsm
, now
, S_RECONNECT
);
320 /* Tell 'fsm' that the connection dropped or that a connection attempt failed.
321 * 'error' specifies the reason: a positive value represents an errno value,
322 * EOF indicates that the connection was closed by the peer (e.g. read()
323 * returned 0), and 0 indicates no specific error.
325 * The FSM will back off, then reconnect. */
327 reconnect_disconnected(struct reconnect
*fsm
, long long int now
, int error
)
329 if (!(fsm
->state
& (S_BACKOFF
| S_VOID
))) {
330 /* Report what happened. */
331 if (fsm
->state
& (S_ACTIVE
| S_IDLE
)) {
333 VLOG_WARN("%s: connection dropped (%s)",
334 fsm
->name
, ovs_strerror(error
));
335 } else if (error
== EOF
) {
336 VLOG(fsm
->info
, "%s: connection closed by peer", fsm
->name
);
338 VLOG(fsm
->info
, "%s: connection dropped", fsm
->name
);
340 } else if (fsm
->state
== S_LISTENING
) {
342 VLOG_WARN("%s: error listening for connections (%s)",
343 fsm
->name
, ovs_strerror(error
));
345 VLOG(fsm
->info
, "%s: error listening for connections",
349 const char *type
= fsm
->passive
? "listen" : "connection";
351 VLOG_INFO("%s: %s attempt failed (%s)",
352 fsm
->name
, type
, ovs_strerror(error
));
354 VLOG(fsm
->info
, "%s: %s attempt timed out", fsm
->name
, type
);
358 if (fsm
->state
& (S_ACTIVE
| S_IDLE
)) {
359 fsm
->last_disconnected
= now
;
362 if (fsm
->state
& (S_ACTIVE
| S_IDLE
)
363 && (fsm
->last_activity
- fsm
->last_connected
>= fsm
->backoff
365 fsm
->backoff
= fsm
->passive
? 0 : fsm
->min_backoff
;
367 if (fsm
->backoff
< fsm
->min_backoff
) {
368 fsm
->backoff
= fsm
->min_backoff
;
369 } else if (fsm
->backoff
>= fsm
->max_backoff
/ 2) {
370 fsm
->backoff
= fsm
->max_backoff
;
375 VLOG(fsm
->info
, "%s: waiting %.3g seconds before trying to "
376 "listen again", fsm
->name
, fsm
->backoff
/ 1000.0);
378 VLOG(fsm
->info
, "%s: waiting %.3g seconds before reconnect",
379 fsm
->name
, fsm
->backoff
/ 1000.0);
383 reconnect_transition__(fsm
, now
,
384 reconnect_may_retry(fsm
) ? S_BACKOFF
: S_VOID
);
388 /* Tell 'fsm' that a connection or listening attempt is in progress.
390 * The FSM will start a timer, after which the connection or listening attempt
391 * will be aborted (by returning RECONNECT_DISCONNECT from
392 * reconnect_run()). */
394 reconnect_connecting(struct reconnect
*fsm
, long long int now
)
396 if (fsm
->state
!= S_CONNECTING
) {
398 VLOG(fsm
->info
, "%s: listening...", fsm
->name
);
400 VLOG(fsm
->info
, "%s: connecting...", fsm
->name
);
402 reconnect_transition__(fsm
, now
, S_CONNECTING
);
406 /* Tell 'fsm' that the client is listening for connection attempts. This state
407 * last indefinitely until the client reports some change.
409 * The natural progression from this state is for the client to report that a
410 * connection has been accepted or is in progress of being accepted, by calling
411 * reconnect_connecting() or reconnect_connected().
413 * The client may also report that listening failed (e.g. accept() returned an
414 * unexpected error such as ENOMEM) by calling reconnect_listen_error(), in
415 * which case the FSM will back off and eventually return RECONNECT_CONNECT
416 * from reconnect_run() to tell the client to try listening again. */
418 reconnect_listening(struct reconnect
*fsm
, long long int now
)
420 if (fsm
->state
!= S_LISTENING
) {
421 VLOG(fsm
->info
, "%s: listening...", fsm
->name
);
422 reconnect_transition__(fsm
, now
, S_LISTENING
);
426 /* Tell 'fsm' that the client's attempt to accept a connection failed
427 * (e.g. accept() returned an unexpected error such as ENOMEM).
429 * If the FSM is currently listening (reconnect_listening() was called), it
430 * will back off and eventually return RECONNECT_CONNECT from reconnect_run()
431 * to tell the client to try listening again. If there is an active
432 * connection, this will be delayed until that connection drops. */
434 reconnect_listen_error(struct reconnect
*fsm
, long long int now
, int error
)
436 if (fsm
->state
== S_LISTENING
) {
437 reconnect_disconnected(fsm
, now
, error
);
441 /* Tell 'fsm' that the connection was successful.
443 * The FSM will start the probe interval timer, which is reset by
444 * reconnect_activity(). If the timer expires, a probe will be sent (by
445 * returning RECONNECT_PROBE from reconnect_run()). If the timer expires
446 * again without being reset, the connection will be aborted (by returning
447 * RECONNECT_DISCONNECT from reconnect_run()). */
449 reconnect_connected(struct reconnect
*fsm
, long long int now
)
451 if (!is_connected_state(fsm
->state
)) {
452 reconnect_connecting(fsm
, now
);
454 VLOG(fsm
->info
, "%s: connected", fsm
->name
);
455 reconnect_transition__(fsm
, now
, S_ACTIVE
);
456 fsm
->last_connected
= now
;
460 /* Tell 'fsm' that the connection attempt failed.
462 * The FSM will back off and attempt to reconnect. */
464 reconnect_connect_failed(struct reconnect
*fsm
, long long int now
, int error
)
466 reconnect_connecting(fsm
, now
);
467 reconnect_disconnected(fsm
, now
, error
);
470 /* Tell 'fsm' that some activity has occurred on the connection. This resets
471 * the probe interval timer, so that the connection is known not to be idle. */
473 reconnect_activity(struct reconnect
*fsm
, long long int now
)
475 if (fsm
->state
!= S_ACTIVE
) {
476 reconnect_transition__(fsm
, now
, S_ACTIVE
);
478 fsm
->last_activity
= now
;
482 reconnect_transition__(struct reconnect
*fsm
, long long int now
,
485 if (fsm
->state
== S_CONNECTING
) {
486 fsm
->n_attempted_connections
++;
487 if (state
== S_ACTIVE
) {
488 fsm
->n_successful_connections
++;
491 if (is_connected_state(fsm
->state
) != is_connected_state(state
)) {
492 if (is_connected_state(fsm
->state
)) {
493 fsm
->total_connected_duration
+= now
- fsm
->last_connected
;
498 VLOG_DBG("%s: entering %s", fsm
->name
, reconnect_state_name__(state
));
500 fsm
->state_entered
= now
;
504 reconnect_deadline__(const struct reconnect
*fsm
)
506 ovs_assert(fsm
->state_entered
!= LLONG_MIN
);
507 switch (fsm
->state
) {
513 return fsm
->state_entered
+ fsm
->backoff
;
516 return fsm
->state_entered
+ MAX(1000, fsm
->backoff
);
519 if (fsm
->probe_interval
) {
520 long long int base
= MAX(fsm
->last_activity
, fsm
->state_entered
);
521 return base
+ fsm
->probe_interval
;
526 if (fsm
->probe_interval
) {
527 return fsm
->state_entered
+ fsm
->probe_interval
;
532 return fsm
->state_entered
;
538 /* Assesses whether any action should be taken on 'fsm'. The return value is
541 * - 0: The client need not take any action.
543 * - Active client, RECONNECT_CONNECT: The client should start a connection
544 * attempt and indicate this by calling reconnect_connecting(). If the
545 * connection attempt has definitely succeeded, it should call
546 * reconnect_connected(). If the connection attempt has definitely
547 * failed, it should call reconnect_connect_failed().
549 * The FSM is smart enough to back off correctly after successful
550 * connections that quickly abort, so it is OK to call
551 * reconnect_connected() after a low-level successful connection
552 * (e.g. connect()) even if the connection might soon abort due to a
553 * failure at a high-level (e.g. SSL negotiation failure).
555 * - Passive client, RECONNECT_CONNECT: The client should try to listen for
556 * a connection, if it is not already listening. It should call
557 * reconnect_listening() if successful, otherwise reconnect_connecting()
558 * or reconnected_connect_failed() if the attempt is in progress or
559 * definitely failed, respectively.
561 * A listening passive client should constantly attempt to accept a new
562 * connection and report an accepted connection with
563 * reconnect_connected().
565 * - RECONNECT_DISCONNECT: The client should abort the current connection
566 * or connection attempt or listen attempt and call
567 * reconnect_disconnected() or reconnect_connect_failed() to indicate it.
569 * - RECONNECT_PROBE: The client should send some kind of request to the
570 * peer that will elicit a response, to ensure that the connection is
571 * indeed in working order. (This will only be returned if the "probe
572 * interval" is nonzero--see reconnect_set_probe_interval()).
574 enum reconnect_action
575 reconnect_run(struct reconnect
*fsm
, long long int now
)
577 if (now
>= reconnect_deadline__(fsm
)) {
578 switch (fsm
->state
) {
583 return RECONNECT_CONNECT
;
586 return RECONNECT_DISCONNECT
;
589 VLOG_DBG("%s: idle %lld ms, sending inactivity probe", fsm
->name
,
590 now
- MAX(fsm
->last_activity
, fsm
->state_entered
));
591 reconnect_transition__(fsm
, now
, S_IDLE
);
592 return RECONNECT_PROBE
;
595 VLOG_ERR("%s: no response to inactivity probe after %.3g "
596 "seconds, disconnecting",
597 fsm
->name
, (now
- fsm
->state_entered
) / 1000.0);
598 return RECONNECT_DISCONNECT
;
601 return RECONNECT_DISCONNECT
;
613 /* Causes the next call to poll_block() to wake up when reconnect_run() should
614 * be called on 'fsm'. */
616 reconnect_wait(struct reconnect
*fsm
, long long int now
)
618 int timeout
= reconnect_timeout(fsm
, now
);
620 poll_timer_wait(timeout
);
624 /* Returns the number of milliseconds after which reconnect_run() should be
625 * called on 'fsm' if nothing else notable happens in the meantime, or a
626 * negative number if this is currently unnecessary. */
628 reconnect_timeout(struct reconnect
*fsm
, long long int now
)
630 long long int deadline
= reconnect_deadline__(fsm
);
631 if (deadline
!= LLONG_MAX
) {
632 long long int remaining
= deadline
- now
;
633 return MAX(0, MIN(INT_MAX
, remaining
));
638 /* Returns true if 'fsm' is currently believed to be connected, that is, if
639 * reconnect_connected() was called more recently than any call to
640 * reconnect_connect_failed() or reconnect_disconnected() or
641 * reconnect_disable(), and false otherwise. */
643 reconnect_is_connected(const struct reconnect
*fsm
)
645 return is_connected_state(fsm
->state
);
648 /* Returns the number of milliseconds since 'fsm' last successfully connected
649 * to its peer (even if it has since disconnected). Returns UINT_MAX if never
652 reconnect_get_last_connect_elapsed(const struct reconnect
*fsm
,
655 return fsm
->last_connected
== LLONG_MAX
? UINT_MAX
656 : now
- fsm
->last_connected
;
659 /* Returns the number of milliseconds since 'fsm' last disconnected
660 * from its peer (even if it has since reconnected). Returns UINT_MAX if never
663 reconnect_get_last_disconnect_elapsed(const struct reconnect
*fsm
,
666 return fsm
->last_disconnected
== LLONG_MAX
? UINT_MAX
667 : now
- fsm
->last_disconnected
;
670 /* Copies various statistics for 'fsm' into '*stats'. */
672 reconnect_get_stats(const struct reconnect
*fsm
, long long int now
,
673 struct reconnect_stats
*stats
)
675 stats
->creation_time
= fsm
->creation_time
;
676 stats
->last_activity
= fsm
->last_activity
;
677 stats
->last_connected
= fsm
->last_connected
;
678 stats
->last_disconnected
= fsm
->last_disconnected
;
679 stats
->backoff
= fsm
->backoff
;
680 stats
->seqno
= fsm
->seqno
;
681 stats
->is_connected
= reconnect_is_connected(fsm
);
682 stats
->msec_since_connect
683 = reconnect_get_last_connect_elapsed(fsm
, now
);
684 stats
->msec_since_disconnect
685 = reconnect_get_last_disconnect_elapsed(fsm
, now
);
686 stats
->total_connected_duration
= fsm
->total_connected_duration
687 + (is_connected_state(fsm
->state
)
688 ? reconnect_get_last_connect_elapsed(fsm
, now
) : 0);
689 stats
->n_attempted_connections
= fsm
->n_attempted_connections
;
690 stats
->n_successful_connections
= fsm
->n_successful_connections
;
691 stats
->state
= reconnect_state_name__(fsm
->state
);
692 stats
->state_elapsed
= now
- fsm
->state_entered
;
696 reconnect_may_retry(struct reconnect
*fsm
)
698 bool may_retry
= fsm
->max_tries
> 0;
699 if (may_retry
&& fsm
->max_tries
!= UINT_MAX
) {