Handle delayed multicast packets that occur with switches

Some switches delay multicast packets vs the unicast token.  This patch works
around that problem by providing a new tuneable called miss_count_const.  This
tuneable works by counting the number of times a message is found missing
and once reaching the const value, marks it as missing in the retransmit list.

This improves performance and doesn't display warning messages about missed
multicast messages when operating in these switching environments.

Signed-off-by: Steven Dake <sdake@redhat.com>
Reviewed-by: Angus Salkeld <asalkeld@redhat.com>

diff --git a/exec/totemconfig.c b/exec/totemconfig.c
index de1d1e4b00fb14c64ee9516e9e8a5209f485ed38..6bb4894b151b4c19b98c6411ea1ff224b684c5c4 100644 (file)
--- a/exec/totemconfig.c
+++ b/exec/totemconfig.c
@@ -79,6 +79,7 @@
 #define MAX_NETWORK_DELAY                      50
 #define WINDOW_SIZE                            50
 #define MAX_MESSAGES                           17
+#define MISS_COUNT_CONST                       5
 #define RRP_PROBLEM_COUNT_TIMEOUT              2000
 #define RRP_PROBLEM_COUNT_THRESHOLD_DEFAULT    10
 #define RRP_PROBLEM_COUNT_THRESHOLD_MIN                5
@@ -219,6 +220,8 @@ static void totem_volatile_config_read (
        objdb_get_string (objdb, object_totem_handle, "vsftype", &totem_config->vsf_type);
 
        objdb_get_int (objdb,object_totem_handle, "max_messages", &totem_config->max_messages);
+
+       objdb_get_int (objdb,object_totem_handle, "miss_count_const", &totem_config->miss_count_const);
 }
 
 
@@ -554,6 +557,10 @@ int totem_config_validate (
                totem_config->max_messages = MAX_MESSAGES;
        }
 
+       if (totem_config->miss_count_const == 0) {
+               totem_config->miss_count_const = MISS_COUNT_CONST;
+       }
+
        if (totem_config->token_timeout < MINIMUM_TIMEOUT) {
                snprintf (local_error_reason, sizeof(local_error_reason),
                        "The token timeout parameter (%d ms) may not be less then (%d ms).",

diff --git a/exec/totemsrp.c b/exec/totemsrp.c
index c9ad3917d9c9391464f2587e3e0c36168d6b2a31..d007fe721a7f9e99d40d944352e61ed49a8bc061 100644 (file)
--- a/exec/totemsrp.c
+++ b/exec/totemsrp.c
@@ -841,6 +841,10 @@ int totemsrp_initialize (
                "window size per rotation (%d messages) maximum messages per rotation (%d messages)\n",
                totem_config->window_size, totem_config->max_messages);
 
+       log_printf (instance->totemsrp_log_level_debug,
+               "missed count const (%d messages)\n",
+               totem_config->miss_count_const);
+
        log_printf (instance->totemsrp_log_level_debug,
                "send threads (%d threads)\n", totem_config->threads);
        log_printf (instance->totemsrp_log_level_debug,
@@ -2450,7 +2454,7 @@ static int orf_token_rtr (
                        strcat (retransmit_msg, value);
                }
                strcat (retransmit_msg, "\n");
-               log_printf (instance->totemsrp_log_level_debug,
+               log_printf (instance->totemsrp_log_level_notice,
                        "%s", retransmit_msg);
        }
 
@@ -2515,6 +2519,20 @@ static int orf_token_rtr (
                 */
                res = sq_item_inuse (sort_queue, instance->my_aru + i);
                if (res == 0) {
+                       /*
+                        * Determine how many times we have missed receiving
+                        * this sequence number.  sq_item_miss_count increments
+                        * a counter for the sequence number.  The miss count
+                        * will be returned and compared.  This allows time for
+                        * delayed multicast messages to be received before
+                        * declaring the message is missing and requesting a
+                        * retransmit.
+                        */
+                       res = sq_item_miss_count (sort_queue, instance->my_aru + i);
+                       if (res < instance->totem_config->miss_count_const) {
+                               continue;
+                       }
+
                        /*
                         * Determine if missing message is already in retransmit list
                         */

diff --git a/include/corosync/sq.h b/include/corosync/sq.h
index aa5bc1e0d72c9f3d322917b01602fc2c3a6ebcc4..ce3b1f8674a0461f36b9819128c3a0658b279616 100644 (file)
--- a/include/corosync/sq.h
+++ b/include/corosync/sq.h
@@ -42,6 +42,7 @@ struct sq {
        unsigned int size;
        void *items;
        unsigned int *items_inuse;
+       unsigned int *items_miss_count;
        unsigned int size_per_item;
        unsigned int head_seqid;
        unsigned int item_count;
@@ -112,7 +113,12 @@ static inline int sq_init (
            == NULL) {
                return (-ENOMEM);
        }
+       if ((sq->items_miss_count = malloc (item_count * sizeof (unsigned int)))
+           == NULL) {
+               return (-ENOMEM);
+       }
        memset (sq->items_inuse, 0, item_count * sizeof (unsigned int));
+       memset (sq->items_miss_count, 0, item_count * sizeof (unsigned int));
        return (0);
 }
 
@@ -124,6 +130,7 @@ static inline void sq_reinit (struct sq *sq, unsigned int head_seqid)
 
        memset (sq->items, 0, sq->item_count * sq->size_per_item);
        memset (sq->items_inuse, 0, sq->item_count * sizeof (unsigned int));
+       memset (sq->items_miss_count, 0, sq->item_count * sizeof (unsigned int));
 }
 
 static inline void sq_assert (const struct sq *sq, unsigned int pos)
@@ -149,11 +156,14 @@ static inline void sq_copy (struct sq *sq_dest, const struct sq *sq_src)
                sq_src->item_count * sq_src->size_per_item);
        memcpy (sq_dest->items_inuse, sq_src->items_inuse,
                sq_src->item_count * sizeof (unsigned int));
+       memcpy (sq_dest->items_miss_count, sq_src->items_miss_count,
+               sq_src->item_count * sizeof (unsigned int));
 }
 
 static inline void sq_free (struct sq *sq) {
        free (sq->items);
        free (sq->items_inuse);
+       free (sq->items_miss_count);
 }
 
 static inline void *sq_item_add (
@@ -178,6 +188,7 @@ static inline void *sq_item_add (
        } else {
                sq->items_inuse[sq_position] = seqid;
        }
+       sq->items_miss_count[sq_position] = 0;
 
        return (sq_item);
 }
@@ -204,6 +215,17 @@ static inline unsigned int sq_item_inuse (
        return (sq->items_inuse[sq_position] != 0);
 }
 
+static inline unsigned int sq_item_miss_count (
+       const struct sq *sq,
+       unsigned int seq_id)
+{
+       unsigned int sq_position;
+
+       sq_position = (sq->head - sq->head_seqid + seq_id) % sq->size;
+       sq->items_miss_count[sq_position]++;
+       return (sq->items_miss_count[sq_position]);
+}
+
 static inline unsigned int sq_size_get (
        const struct sq *sq)
 {
@@ -286,6 +308,8 @@ static inline void sq_items_release (struct sq *sq, unsigned int seqid)
 //             printf ("releasing %d for %d\n", oldhead, seqid - sq->head_seqid + 1);
                memset (&sq->items_inuse[oldhead], 0,
                        (seqid - sq->head_seqid + 1) * sizeof (unsigned int));
+               memset (&sq->items_miss_count[oldhead], 0,
+                       (seqid - sq->head_seqid + 1) * sizeof (unsigned int));
        }
        sq->head_seqid = seqid + 1;
 }

diff --git a/include/corosync/totem/totem.h b/include/corosync/totem/totem.h
index cf78e4c096b064ef19233147f19841dabc416163..1c631866d51557d36e22070bf2e89bbe99613211 100644 (file)
--- a/include/corosync/totem/totem.h
+++ b/include/corosync/totem/totem.h
@@ -180,6 +180,8 @@ struct totem_config {
        int crypto_sign_type;
 
        totem_transport_t transport_number;
+
+       unsigned int miss_count_const;
 };
 
 #define TOTEM_CONFIGURATION_TYPE
@@ -264,6 +266,9 @@ typedef struct {
 
 } totemsrp_stats_t;
 
+ 
+ #define TOTEM_CONFIGURATION_TYPE
+
 typedef struct {
        totem_stats_header_t hdr;
        totemsrp_stats_t *srp;

diff --git a/man/corosync.conf.5 b/man/corosync.conf.5
index 79fb5a2ddd0bb94eeb0ceeabeff1ab1c7812be76..a017ea3731630c1993ee78580f9d48434ad5a675 100644 (file)
--- a/man/corosync.conf.5
+++ b/man/corosync.conf.5
@@ -437,6 +437,16 @@ processor on receipt of the token.  The max_messages parameter is limited to
 
 The default is 17 messages.
 
+.TP
+miss_count_const
+This constant defines the maximum number of times on receipt of a token
+a message is checked for retransmission before a retransmission occurs.  This
+parameter is useful to modify for switches that delay multicast packets
+compared to unicast packets.  The default setting works well for nearly all
+modern switches.
+
+The default is 5 messages.
+
 .TP
 rrp_problem_count_timeout
 This specifies the time in milliseconds to wait before decrementing the