Move ringid store and load from totem library

Functions for storing and loading ring id was in the totem library. This
causes problem, what to do when it's impossible to load or store ring
id. Easy solution seemed to be assert, but sadly this makes hard for
user to find out what happened (because corosync was just aborted and
logsys didn't flush)

Solution is to move these functions to main.c, where is much easier to
handle error. This also makes libtotem free of any file system
operations.

Signed-off-by: Jan Friesse <jfriesse@redhat.com>
Reviewed-by: Christine Caulfield <ccaulfie@redhat.com>

diff --git a/exec/main.c b/exec/main.c
index d6300589e5d841509c18002eae5f5076b1b39e67..7b5d73fae755f25c377c08dc748f2a7448044058 100644 (file)
--- a/exec/main.c
+++ b/exec/main.c
@@ -703,6 +703,87 @@ int main_mcast (
        return (totempg_groups_mcast_joined (corosync_group_handle, iovec, iov_len, guarantee));
 }
 
+static void corosync_ring_id_create_or_load (
+       struct memb_ring_id *memb_ring_id,
+       const struct totem_ip_address *addr)
+{
+       int fd;
+       int res = 0;
+       char filename[PATH_MAX];
+
+       snprintf (filename, sizeof(filename), "%s/ringid_%s",
+               get_run_dir(), totemip_print (addr));
+       fd = open (filename, O_RDONLY, 0700);
+       /*
+        * If file can be opened and read, read the ring id
+        */
+       if (fd != -1) {
+               res = read (fd, &memb_ring_id->seq, sizeof (uint64_t));
+               close (fd);
+       }
+       /*
+        * If file could not be opened or read, create a new ring id
+        */
+       if ((fd == -1) || (res != sizeof (uint64_t))) {
+               memb_ring_id->seq = 0;
+               umask(0);
+               fd = open (filename, O_CREAT|O_RDWR, 0700);
+               if (fd != -1) {
+                       res = write (fd, &memb_ring_id->seq, sizeof (uint64_t));
+                       close (fd);
+                       if (res == -1) {
+                               LOGSYS_PERROR (errno, LOGSYS_LEVEL_ERROR,
+                                       "Couldn't write ringid file '%s'", filename);
+
+                               corosync_exit_error (COROSYNC_DONE_STORE_RINGID);
+                       }
+               } else {
+                       LOGSYS_PERROR (errno, LOGSYS_LEVEL_ERROR,
+                               "Couldn't create ringid file '%s'", filename);
+
+                       corosync_exit_error (COROSYNC_DONE_STORE_RINGID);
+               }
+       }
+
+       totemip_copy(&memb_ring_id->rep, addr);
+       assert (!totemip_zero_check(&memb_ring_id->rep));
+}
+
+static void corosync_ring_id_store (
+       const struct memb_ring_id *memb_ring_id,
+       const struct totem_ip_address *addr)
+{
+       char filename[PATH_MAX];
+       int fd;
+       int res;
+
+       snprintf (filename, sizeof(filename), "%s/ringid_%s",
+               get_run_dir(), totemip_print (addr));
+
+       fd = open (filename, O_WRONLY, 0777);
+       if (fd == -1) {
+               fd = open (filename, O_CREAT|O_RDWR, 0777);
+       }
+       if (fd == -1) {
+               LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR,
+                       "Couldn't store new ring id %llx to stable storage",
+                       memb_ring_id->seq);
+
+               corosync_exit_error (COROSYNC_DONE_STORE_RINGID);
+       }
+       log_printf (LOGSYS_LEVEL_DEBUG,
+               "Storing new sequence id for ring %llx", memb_ring_id->seq);
+       res = write (fd, &memb_ring_id->seq, sizeof(memb_ring_id->seq));
+       close (fd);
+       if (res != sizeof(memb_ring_id->seq)) {
+               LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR,
+                       "Couldn't store new ring id %llx to stable storage",
+                       memb_ring_id->seq);
+
+               corosync_exit_error (COROSYNC_DONE_STORE_RINGID);
+       }
+}
+
 static qb_loop_timer_handle recheck_the_q_level_timer;
 void corosync_recheck_the_q_level(void *data)
 {
@@ -1231,6 +1312,9 @@ int main (int argc, char **argv, char **envp)
 
        ip_version = totem_config.ip_version;
 
+       totem_config.totem_memb_ring_id_create_or_load = corosync_ring_id_create_or_load;
+       totem_config.totem_memb_ring_id_store = corosync_ring_id_store;
+
        totem_config.totem_logging_configuration = totem_logging_configuration;
        totem_config.totem_logging_configuration.log_subsys_id = _logsys_subsys_create("TOTEM", "totem,"
                        "totemmrp.c,totemrrp.c,totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"

diff --git a/exec/totemsrp.c b/exec/totemsrp.c
index dc6b2099d8ad17638fb4d93b883a6aa995e4ed7a..dcda8d1a4c0f62f1fe1bae99a43f3ab7845db5cb 100644 (file)
--- a/exec/totemsrp.c
+++ b/exec/totemsrp.c
@@ -91,7 +91,6 @@
 #include "totemnet.h"
 
 #include "cs_queue.h"
-#include "util.h"
 
 #define LOCALHOST_IP                           inet_addr("127.0.0.1")
 #define QUEUE_RTR_ITEMS_SIZE_MAX               16384 /* allow 16384 retransmit items */
@@ -467,6 +466,14 @@ struct totemsrp_instance {
        void (*totemsrp_waiting_trans_ack_cb_fn) (
                int waiting_trans_ack);
 
+       void (*memb_ring_id_create_or_load) (
+               struct memb_ring_id *memb_ring_id,
+               const struct totem_ip_address *addr);
+
+       void (*memb_ring_id_store) (
+               const struct memb_ring_id *memb_ring_id,
+               const struct totem_ip_address *addr);
+
        int global_seqno;
 
        int my_token_held;
@@ -623,8 +630,6 @@ static int srp_addr_equal (const struct srp_addr *a, const struct srp_addr *b);
 
 static void memb_leave_message_send (struct totemsrp_instance *instance);
 
-static void memb_ring_id_create_or_load (struct totemsrp_instance *, struct memb_ring_id *);
-
 static void token_callbacks_execute (struct totemsrp_instance *instance, enum totem_callback_token_type type);
 static void memb_state_gather_enter (struct totemsrp_instance *instance, enum gather_state_from gather_from);
 static void messages_deliver_to_app (struct totemsrp_instance *instance, int skip, unsigned int end_point);
@@ -632,7 +637,7 @@ static int orf_token_mcast (struct totemsrp_instance *instance, struct orf_token
        int fcc_mcasts_allowed);
 static void messages_free (struct totemsrp_instance *instance, unsigned int token_aru);
 
-static void memb_ring_id_set_and_store (struct totemsrp_instance *instance,
+static void memb_ring_id_set (struct totemsrp_instance *instance,
        const struct memb_ring_id *ring_id);
 static void target_set_completed (void *context);
 static void memb_state_commit_token_update (struct totemsrp_instance *instance);
@@ -871,6 +876,12 @@ int totemsrp_initialize (
        instance->totemsrp_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
        instance->totemsrp_log_printf = totem_config->totem_logging_configuration.log_printf;
 
+       /*
+        * Configure totem store and load functions
+        */
+       instance->memb_ring_id_create_or_load = totem_config->totem_memb_ring_id_create_or_load;
+       instance->memb_ring_id_store = totem_config->totem_memb_ring_id_store;
+
        /*
         * Initialize local variables for totemsrp
         */
@@ -2107,7 +2118,8 @@ static void memb_state_commit_enter (
 
        instance->memb_timer_state_gather_consensus_timeout = 0;
 
-       memb_ring_id_set_and_store (instance, &instance->commit_token->ring_id);
+       memb_ring_id_set (instance, &instance->commit_token->ring_id);
+       instance->memb_ring_id_store (&instance->my_ring_id, &instance->my_id.addr[0]);
 
        instance->token_ring_id_seq = instance->my_ring_id.seq;
 
@@ -3292,79 +3304,12 @@ static void memb_merge_detect_transmit (struct totemsrp_instance *instance)
                sizeof (struct memb_merge_detect));
 }
 
-static void memb_ring_id_create_or_load (
-       struct totemsrp_instance *instance,
-       struct memb_ring_id *memb_ring_id)
-{
-       int fd;
-       int res = 0;
-       char filename[PATH_MAX];
-
-       snprintf (filename, sizeof(filename), "%s/ringid_%s",
-               get_run_dir(), totemip_print (&instance->my_id.addr[0]));
-       fd = open (filename, O_RDONLY, 0700);
-       /*
-        * If file can be opened and read, read the ring id
-        */
-       if (fd != -1) {
-               res = read (fd, &memb_ring_id->seq, sizeof (uint64_t));
-               close (fd);
-       }
-       /*
-        * If file could not be opened or read, create a new ring id
-        */
-       if ((fd == -1) || (res != sizeof (uint64_t))) {
-               memb_ring_id->seq = 0;
-               umask(0);
-               fd = open (filename, O_CREAT|O_RDWR, 0700);
-               if (fd != -1) {
-                       res = write (fd, &memb_ring_id->seq, sizeof (uint64_t));
-                       close (fd);
-                       if (res == -1) {
-                               LOGSYS_PERROR (errno, instance->totemsrp_log_level_warning,
-                                       "Couldn't write ringid file '%s'", filename);
-                       }
-               } else {
-                       LOGSYS_PERROR (errno, instance->totemsrp_log_level_warning,
-                               "Couldn't create ringid file '%s'", filename);
-               }
-       }
-
-       totemip_copy(&memb_ring_id->rep, &instance->my_id.addr[0]);
-       assert (!totemip_zero_check(&memb_ring_id->rep));
-       instance->token_ring_id_seq = memb_ring_id->seq;
-}
-
-static void memb_ring_id_set_and_store (
+static void memb_ring_id_set (
        struct totemsrp_instance *instance,
        const struct memb_ring_id *ring_id)
 {
-       char filename[256];
-       int fd;
-       int res;
 
        memcpy (&instance->my_ring_id, ring_id, sizeof (struct memb_ring_id));
-
-       snprintf (filename, sizeof(filename), "%s/ringid_%s",
-               get_run_dir(), totemip_print (&instance->my_id.addr[0]));
-
-       fd = open (filename, O_WRONLY, 0777);
-       if (fd == -1) {
-               fd = open (filename, O_CREAT|O_RDWR, 0777);
-       }
-       if (fd == -1) {
-               LOGSYS_PERROR(errno, instance->totemsrp_log_level_warning,
-                       "Couldn't store new ring id %llx to stable storage",
-                       instance->my_ring_id.seq);
-               assert (0);
-               return;
-       }
-       log_printf (instance->totemsrp_log_level_debug,
-               "Storing new sequence id for ring %llx", instance->my_ring_id.seq);
-       //assert (fd > 0);
-       res = write (fd, &instance->my_ring_id.seq, sizeof (unsigned long long));
-       assert (res == sizeof (unsigned long long));
-       close (fd);
 }
 
 int totemsrp_callback_token_create (
@@ -4641,7 +4586,9 @@ void main_iface_change_fn (
        totemip_copy (&instance->my_memb_list[0].addr[iface_no], iface_addr);
 
        if (instance->iface_changes++ == 0) {
-               memb_ring_id_create_or_load (instance, &instance->my_ring_id);
+               instance->memb_ring_id_create_or_load (&instance->my_ring_id,
+                   &instance->my_id.addr[0]);
+               instance->token_ring_id_seq = instance->my_ring_id.seq;
                log_printf (
                        instance->totemsrp_log_level_debug,
                        "Created or loaded sequence id %llx.%s for this ring.",

diff --git a/exec/util.h b/exec/util.h
index 63caf74f14d6ea4bff40556645f6fe0af10a8cd4..9efd35e50f7b5e348efcba110ad6896d17ea494e 100644 (file)
--- a/exec/util.h
+++ b/exec/util.h
@@ -58,6 +58,7 @@ enum e_corosync_done {
        COROSYNC_DONE_ALREADY_RUNNING = 18,
        COROSYNC_DONE_STD_TO_NULL_REDIR = 19,
        COROSYNC_DONE_SERVICE_ENGINE_INIT = 20,
+       COROSYNC_DONE_STORE_RINGID = 21,
        COROSYNC_DONE_PLOAD = 99
 };
 

diff --git a/include/corosync/totem/totem.h b/include/corosync/totem/totem.h
index 72a3e1a884e135c864776e961dde295ff7ef52fa..7f9fb0f9ba492bc5d02807462bfe4af7baea300c 100644 (file)
--- a/include/corosync/totem/totem.h
+++ b/include/corosync/totem/totem.h
@@ -99,6 +99,12 @@ typedef enum {
        TOTEM_TRANSPORT_RDMA = 2
 } totem_transport_t;
 
+#define MEMB_RING_ID
+struct memb_ring_id {
+       struct totem_ip_address rep;
+       unsigned long long seq;
+} __attribute__((packed));
+
 struct totem_config {
        int version;
 
@@ -181,6 +187,14 @@ struct totem_config {
        unsigned int miss_count_const;
 
        int ip_version;
+
+       void (*totem_memb_ring_id_create_or_load) (
+           struct memb_ring_id *memb_ring_id,
+           const struct totem_ip_address *addr);
+
+       void (*totem_memb_ring_id_store) (
+           const struct memb_ring_id *memb_ring_id,
+           const struct totem_ip_address *addr);
 };
 
 #define TOTEM_CONFIGURATION_TYPE
@@ -200,12 +214,6 @@ enum totem_event_type {
        TOTEM_EVENT_NEW_MSG,
 };
 
-#define MEMB_RING_ID
-struct memb_ring_id {
-       struct totem_ip_address rep;
-       unsigned long long seq;
-} __attribute__((packed));
-
 typedef struct {
        int is_dirty;
        time_t last_updated;