knet: Fix MTU sizes & allow transport config in corosync.conf

Corosync layers don't need to know the knet MTU size - this way
corosync fragments buffers only when they get larger than the
KNET buffer size (64K) and knet fragments below that based on
the actual MTU and transport considerations.

It is also now possible to configure knet to use UDP or SCTP
transports in corosync.conf. This is currently done per-link
so if you have more than 1 link you need several interface{}
stanzas inside totem{} to make it use other than the default
of UDP. if it's useful I might add the option of a global
default.

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>

diff --git a/exec/coroparse.c b/exec/coroparse.c
index b660335d6b31d8c696c0a753480ef97a2e0d9a86..44ca7b93710535e4697c075d30ca82c8ed314f85 100644 (file)
--- a/exec/coroparse.c
+++ b/exec/coroparse.c
@@ -120,6 +120,7 @@ struct main_cp_cb_data {
        int knet_ping_precision;
        int knet_pong_count;
        int knet_pmtud_interval;
+       char *knet_transport;
 
        struct qb_list_head logger_subsys_items_head;
        char *subsys;
@@ -766,6 +767,11 @@ static int main_config_parser_cb(const char *path,
                                data->knet_pong_count = val;
                                add_as_string = 0;
                        }
+                       if (strcmp(path, "totem.interface.knet_transport") == 0) {
+                               val_type = ICMAP_VALUETYPE_STRING;
+                               data->knet_transport = strdup(value);
+                               add_as_string = 0;
+                       }
                        break;
                case MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS:
                        if (strcmp(key, "subsys") == 0) {
@@ -958,6 +964,7 @@ static int main_config_parser_cb(const char *path,
                        data->knet_ping_timeout = -1;
                        data->knet_ping_precision = -1;
                        data->knet_pong_count = -1;
+                       data->knet_transport = NULL;
                        qb_list_init(&data->member_items_head);
                };
                if (strcmp(path, "totem") == 0) {
@@ -1082,6 +1089,12 @@ static int main_config_parser_cb(const char *path,
                                                data->linknumber);
                                icmap_set_uint32_r(config_map, key_name, data->knet_pong_count);
                        }
+                       if (data->knet_transport) {
+                               snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_transport",
+                                               data->linknumber);
+                               icmap_set_string_r(config_map, key_name, data->knet_transport);
+                               free(data->knet_transport);
+                       }
 
                        ii = 0;
 

diff --git a/exec/totemconfig.c b/exec/totemconfig.c
index d74f7e5cfd8e7aeae8eeb659bb03d30bc00f28e5..88e2b684729ca44e0c187180ca49b982bff55793 100644 (file)
--- a/exec/totemconfig.c
+++ b/exec/totemconfig.c
@@ -52,6 +52,7 @@
 #include <corosync/swab.h>
 #include <qb/qblist.h>
 #include <qb/qbdefs.h>
+#include <libknet.h>
 #include <corosync/totem/totem.h>
 #include <corosync/config.h>
 #include <corosync/logsys.h>
@@ -80,6 +81,7 @@
 #define KNET_PING_PRECISION                     2048
 #define KNET_PONG_COUNT                         2
 #define KNET_PMTUD_INTERVAL                     30
+#define KNET_DEFAULT_TRANSPORT                  KNET_TRANSPORT_UDP
 
 #define DEFAULT_PORT                           5405
 
@@ -1137,6 +1139,21 @@ extern int totem_config_read (
                        totem_config->interfaces[linknumber].knet_pong_count = u32;
                }
 
+               totem_config->interfaces[linknumber].knet_transport = KNET_DEFAULT_TRANSPORT;
+               snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_transport", linknumber);
+               if (icmap_get_string(tmp_key, &str) == CS_OK) {
+                       if (strcmp(str, "sctp") == 0) {
+                               totem_config->interfaces[linknumber].knet_transport = KNET_TRANSPORT_SCTP;
+                       }
+                       else if (strcmp(str, "udp") == 0) {
+                               totem_config->interfaces[linknumber].knet_transport = KNET_TRANSPORT_UDP;
+                       }
+                       else {
+                               *error_string = "Unrecognised knet_transport. expected 'udp' or 'sctp'";
+                               return -1;
+                       }
+               }
+
                snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.member.", linknumber);
                member_iter = icmap_iter_init(tmp_key);
                while ((member_iter_key = icmap_iter_next(member_iter, NULL, NULL)) != NULL) {
@@ -1400,7 +1417,12 @@ int totem_config_validate (
        }
 
        if (totem_config->net_mtu == 0) {
-               totem_config->net_mtu = 1500;
+               if (totem_config->transport_number == TOTEM_TRANSPORT_KNET) {
+                       totem_config->net_mtu = KNET_MAX_PACKET_SIZE;
+               }
+               else {
+                       totem_config->net_mtu = 1500;
+               }
        }
 
        return 0;

diff --git a/exec/totemknet.c b/exec/totemknet.c
index 76674778ef664e21dd1c29846fa3f5699b23fe1e..c43b8458b7543370ff56d547b0627f2c0cf5358a 100644 (file)
--- a/exec/totemknet.c
+++ b/exec/totemknet.c
@@ -282,8 +282,8 @@ static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
        struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
        knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
 
-       // TODO: Check this
-       instance->totemknet_mtu_changed(instance->context, data_mtu);
+       /* We don't need to tell corosync the actual knet MTU */
+//     instance->totemknet_mtu_changed(instance->context, data_mtu);
 }
 
 int totemknet_crypto_set (
@@ -918,7 +918,7 @@ int totemknet_initialize (
 
 void *totemknet_buffer_alloc (void)
 {
-       return malloc (FRAME_SIZE_MAX);
+       return malloc(KNET_MAX_PACKET_SIZE);
 }
 
 void totemknet_buffer_release (void *ptr)
@@ -1111,7 +1111,9 @@ int totemknet_member_add (
        /* Casts to remove const */
        totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port+link_no, &remote_ss, &addrlen);
        totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port+link_no, &local_ss, &addrlen);
-       err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no, KNET_TRANSPORT_UDP, &local_ss, &remote_ss);
+       err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
+                                  instance->totem_config->interfaces[link_no].knet_transport,
+                                  &local_ss, &remote_ss);
        if (err) {
                KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
                return -1;

diff --git a/include/corosync/totem/totem.h b/include/corosync/totem/totem.h
index 37468010dc78331583c4c983c6f992db91d8a196..5a67a454bc29de9d0b4375672c9798f1dedc8898 100644 (file)
--- a/include/corosync/totem/totem.h
+++ b/include/corosync/totem/totem.h
@@ -74,6 +74,7 @@ struct totem_interface {
        int knet_ping_timeout;
        int knet_ping_precision;
        int knet_pong_count;
+       int knet_transport;
        struct totem_ip_address member_list[PROCESSOR_COUNT_MAX];
 };
 

diff --git a/man/corosync.conf.5 b/man/corosync.conf.5
index 8060e0d1a00e0bde046dfd7d889a45043f1a8267..4a021f987769b401acbd403a38268dd42a4519f2 100644 (file)
--- a/man/corosync.conf.5
+++ b/man/corosync.conf.5
@@ -71,9 +71,6 @@ The
 For knet, multiple interface subsections define parameters for each knet link on the
 system.
 
-For UDP, there should be just one interface section that defines the multicast or
-broadcast options for the link.
-
 For UDPU an interface section is not needed and it is recommended that the nodelist
 is used to define cluster nodes.
 
@@ -107,6 +104,10 @@ the average link latency. (default 2048 samples)
 .TP
 knet_pong_count
 How many valid ping/pongs before a link is marked UP. (default 5)
+.TP
+
+knet_transport
+Which IP transport knet should use. valid values are "sctp" or "udp". (default: udp)
 
 .TP
 bindnetaddr (udp only)