[PMTUd] add dynamic pong timeout when using crypto

problem originally reported by proxmox community, users
observed that under pressure the MTU would flap back and forth
between 2 values due to other node response timeout.

implement a dynamic timeout multiplier when using crypto that
should solve the problem in a more flexible fashion.

When a timeout hits, those new logs will show:

[knet]: [info] host: host: 1 (passive) best link: 0 (pri: 0)
[knet]: [debug] pmtud: Starting PMTUD for host: 1 link: 0
[knet]: [debug] pmtud: Increasing PMTUd response timeout multiplier to (4) for host 1 link: 0
[knet]: [info] pmtud: PMTUD link change for host: 1 link: 0 from 469 to 65429
[knet]: [debug] pmtud: PMTUD completed for host: 1 link: 0 current link mtu: 65429
[knet]: [info] pmtud: Global data MTU changed to: 65429
[knet]: [debug] pmtud: Starting PMTUD for host: 1 link: 0
[knet]: [debug] pmtud: Increasing PMTUd response timeout multiplier to (8) for host 1 link: 0
[knet]: [debug] pmtud: Increasing PMTUd response timeout multiplier to (16) for host 1 link: 0
[knet]: [debug] pmtud: Increasing PMTUd response timeout multiplier to (32) for host 1 link: 0
[knet]: [debug] pmtud: Increasing PMTUd response timeout multiplier to (64) for host 1 link: 0
[knet]: [debug] pmtud: PMTUD completed for host: 1 link: 0 current link mtu: 65429
[knet]: [debug] pmtud: Starting PMTUD for host: 1 link: 0
[knet]: [debug] pmtud: Increasing PMTUd response timeout multiplier to (128) for host 1 link: 0
[knet]: [debug] pmtud: PMTUD completed for host: 1 link: 0 current link mtu: 65429

and when the latency reduces and it is safe to be more responsive again:

[knet]: [debug] pmtud: Starting PMTUD for host: 1 link: 0
[knet]: [debug] pmtud: Decreasing PMTUd response timeout multiplier to (64) for host 1 link: 0
[knet]: [debug] pmtud: PMTUD completed for host: 1 link: 0 current link mtu: 65429

....

testing this patch on normal hosts is a bit challenging tho.

Patch was tested by hardcoding a super low timeout here:

diff --git a/libknet/threads_pmtud.c b/libknet/threads_pmtud.c
index 4f0ba0f..5e2b89b 100644
--- a/libknet/threads_pmtud.c
+++ b/libknet/threads_pmtud.c
@@ -261,7 +271,8 @@ retry:
                        /*
                         * crypto, under pressure, is a royal PITA
                         */
-                       pong_timeout_adj_tmp = dst_link->pong_timeout_adj * 2;
+                       //pong_timeout_adj_tmp = dst_link->pong_timeout_adj * dst_link->pmtud_crypto_timeout_multiplier;
+                       pong_timeout_adj_tmp = 30 * dst_link->pmtud_crypto_timeout_multiplier;
                } else {
                        pong_timeout_adj_tmp = dst_link->pong_timeout_adj;
                }

and using a long running version of api_knet_send_crypto_test with a short PMTUd setfreq (10 sec).

Signed-off-by: Fabio M. Di Nitto <fdinitto@redhat.com>

diff --git a/libknet/internals.h b/libknet/internals.h
index 95450d5f7763ce4256acc55bd2ee103f42750b40..82c3d8195e0873b18bd117ed611f004391037ded 100644 (file)
--- a/libknet/internals.h
+++ b/libknet/internals.h
@@ -82,6 +82,7 @@ struct knet_link {
        uint32_t last_bad_mtu;
        uint32_t last_sent_mtu;
        uint32_t last_recv_mtu;
+       uint32_t pmtud_crypto_timeout_multiplier;/* used by PMTUd to adjust timeouts on high loads */
        uint8_t has_valid_mtu;
 };
 

diff --git a/libknet/links.c b/libknet/links.c
index 0928a38fc919db856a4b40eb42cfe87798429861..6abbd48844f18960a0b40d2d2658dd71fb752b93 100644 (file)
--- a/libknet/links.c
+++ b/libknet/links.c
@@ -238,6 +238,7 @@ int knet_link_set_config(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t l
                }
        }
 
+       link->pmtud_crypto_timeout_multiplier = KNET_LINK_PMTUD_CRYPTO_TIMEOUT_MULTIPLIER_MIN;
        link->pong_count = KNET_LINK_DEFAULT_PONG_COUNT;
        link->has_valid_mtu = 0;
        link->ping_interval = KNET_LINK_DEFAULT_PING_INTERVAL * 1000; /* microseconds */

diff --git a/libknet/links.h b/libknet/links.h
index e14958deef9ad9a2d3fc47cc98e147fdd7e5086c..c8ca610941a98625faff53d7103ef210983c56e2 100644 (file)
--- a/libknet/links.h
+++ b/libknet/links.h
@@ -30,6 +30,16 @@
  */
 #define KNET_LINK_PONG_TIMEOUT_LAT_MUL 2
 
+/*
+ * under heavy load with crypto enabled, it takes much
+ * longer time to receive a response from the other node.
+ *
+ * 128 is somewhat arbitrary number but we want to set a limit
+ * and report failures after that.
+ */
+#define KNET_LINK_PMTUD_CRYPTO_TIMEOUT_MULTIPLIER_MIN    2
+#define KNET_LINK_PMTUD_CRYPTO_TIMEOUT_MULTIPLIER_MAX  128
+
 int _link_updown(knet_handle_t knet_h, knet_node_id_t node_id, uint8_t link_id,
                 unsigned int enabled, unsigned int connected);
 

diff --git a/libknet/threads_pmtud.c b/libknet/threads_pmtud.c
index 4f0ba0f209bd1152b947eccbb2e3f4ecf51ad75d..54aa15272218939ae2928d9ab8bbc56735369ad9 100644 (file)
--- a/libknet/threads_pmtud.c
+++ b/libknet/threads_pmtud.c
@@ -36,8 +36,9 @@ static int _handle_check_link_pmtud(knet_handle_t knet_h, struct knet_host *dst_
        size_t app_mtu_len;             /* real data that we can send onwire */
        ssize_t len;                    /* len of what we were able to sendto onwire */
 
-       struct timespec ts;
-       unsigned long long pong_timeout_adj_tmp;
+       struct timespec ts, pmtud_crypto_start_ts, pmtud_crypto_stop_ts;
+       unsigned long long pong_timeout_adj_tmp, timediff;
+       int pmtud_crypto_reduce = 1;
        unsigned char *outbuf = (unsigned char *)knet_h->pmtudbuf;
 
        warn_once = 0;
@@ -242,6 +243,15 @@ retry:
                        return -1;
                }
 
+               /*
+                * non fatal, we can wait the next round to reduce the
+                * multiplier
+                */
+               if (clock_gettime(CLOCK_MONOTONIC, &pmtud_crypto_start_ts) < 0) {
+                       log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get current time: %s", strerror(errno));
+                       pmtud_crypto_reduce = 0;
+               }
+
                /*
                 * set PMTUd reply timeout to match pong_timeout on a given link
                 *
@@ -261,7 +271,7 @@ retry:
                        /*
                         * crypto, under pressure, is a royal PITA
                         */
-                       pong_timeout_adj_tmp = dst_link->pong_timeout_adj * 2;
+                       pong_timeout_adj_tmp = dst_link->pong_timeout_adj * dst_link->pmtud_crypto_timeout_multiplier;
                } else {
                        pong_timeout_adj_tmp = dst_link->pong_timeout_adj;
                }
@@ -299,6 +309,17 @@ retry:
 
                if (ret) {
                        if (ret == ETIMEDOUT) {
+                               if ((knet_h->crypto_instance) && (dst_link->pmtud_crypto_timeout_multiplier < KNET_LINK_PMTUD_CRYPTO_TIMEOUT_MULTIPLIER_MAX)) {
+                                       dst_link->pmtud_crypto_timeout_multiplier = dst_link->pmtud_crypto_timeout_multiplier * 2;
+                                       pmtud_crypto_reduce = 0;
+                                       log_debug(knet_h, KNET_SUB_PMTUD,
+                                                       "Increasing PMTUd response timeout multiplier to (%u) for host %u link: %u",
+                                                       dst_link->pmtud_crypto_timeout_multiplier,
+                                                       dst_host->host_id,
+                                                       dst_link->link_id);
+                                       pthread_mutex_unlock(&knet_h->pmtud_mutex);
+                                       goto restart;
+                               }
                                if (!warn_once) {
                                        log_warn(knet_h, KNET_SUB_PMTUD,
                                                        "possible MTU misconfiguration detected. "
@@ -327,6 +348,23 @@ retry:
                        }
                }
 
+               if ((knet_h->crypto_instance) && (pmtud_crypto_reduce == 1) &&
+                   (dst_link->pmtud_crypto_timeout_multiplier > KNET_LINK_PMTUD_CRYPTO_TIMEOUT_MULTIPLIER_MIN)) {
+                       if (!clock_gettime(CLOCK_MONOTONIC, &pmtud_crypto_stop_ts)) {
+                               timespec_diff(pmtud_crypto_start_ts, pmtud_crypto_stop_ts, &timediff);
+                               if (((pong_timeout_adj_tmp * 1000) / 2) > timediff) {
+                                       dst_link->pmtud_crypto_timeout_multiplier = dst_link->pmtud_crypto_timeout_multiplier / 2;
+                                       log_debug(knet_h, KNET_SUB_PMTUD,
+                                                       "Decreasing PMTUd response timeout multiplier to (%u) for host %u link: %u",
+                                                       dst_link->pmtud_crypto_timeout_multiplier,
+                                                       dst_host->host_id,
+                                                       dst_link->link_id);
+                               }
+                       } else {
+                               log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get current time: %s", strerror(errno));
+                       }
+               }
+
                if ((dst_link->last_recv_mtu != onwire_len) || (ret)) {
                        dst_link->last_bad_mtu = onwire_len;
                } else {