Fix "leave" function where the remaining nodes recalculate quorum when a

node leaves. Also add a timeout to the leave flag.

git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@1757 fd59a12c-fef9-0310-b244-a6a79926bd2f

diff --git a/services/votequorum.c b/services/votequorum.c
index 00724e110108c6dc0521c6b51725b15834e18ab5..ead0bf3b07fdc3feca7ecdab7db916fb53219f65 100644 (file)
--- a/services/votequorum.c
+++ b/services/votequorum.c
@@ -71,6 +71,7 @@
  /* Silly default to prevent accidents! */
 #define DEFAULT_EXPECTED   1024
 #define DEFAULT_QDEV_POLL 10000
+#define DEFAULT_LEAVE_TMO 10000
 
 LOGSYS_DECLARE_SUBSYS ("VOTEQ", LOG_INFO);
 
@@ -125,11 +126,13 @@ static int quorum;
 static int cluster_is_quorate;
 static int first_trans = 1;
 static unsigned int quorumdev_poll = DEFAULT_QDEV_POLL;
+static unsigned int leaving_timeout = DEFAULT_LEAVE_TMO;
 
 static struct cluster_node *us;
 static struct cluster_node *quorum_device = NULL;
 static char quorum_device_name[VOTEQUORUM_MAX_QDISK_NAME_LEN];
 static corosync_timer_handle_t quorum_device_timer;
+static corosync_timer_handle_t leaving_timer;
 static struct list_head cluster_members_list;
 static struct corosync_api_v1 *corosync_api;
 static struct list_head trackers_list;
@@ -484,6 +487,7 @@ static void read_quorum_config(unsigned int quorum_handle)
        objdb_get_int(corosync_api, quorum_handle, "expected_votes", &us->expected_votes, DEFAULT_EXPECTED);
        objdb_get_int(corosync_api, quorum_handle, "votes", &us->votes, 1);
        objdb_get_int(corosync_api, quorum_handle, "quorumdev_poll", &quorumdev_poll, DEFAULT_QDEV_POLL);
+       objdb_get_int(corosync_api, quorum_handle, "leaving_timeout", &leaving_timeout, DEFAULT_LEAVE_TMO);
        objdb_get_int(corosync_api, quorum_handle, "disallowed", &value, 0);
        if (value)
                quorum_flags |= VOTEQUORUM_FLAG_FEATURE_DISALLOWED;
@@ -667,7 +671,6 @@ static int calculate_quorum(int allow_decrease, int max_expected, unsigned int *
        unsigned int highest_expected = 0;
        unsigned int newquorum, q1, q2;
        unsigned int total_nodes = 0;
-       unsigned int leaving = 0;
 
        ENTER();
        list_iterate(nodelist, &cluster_members_list) {
@@ -684,9 +687,6 @@ static int calculate_quorum(int allow_decrease, int max_expected, unsigned int *
                        total_votes += node->votes;
                        total_nodes++;
                }
-               if (node->state == NODESTATE_LEAVING) {
-                       leaving = 1;
-               }
        }
 
        if (quorum_device && quorum_device->state == NODESTATE_MEMBER)
@@ -723,12 +723,26 @@ static int calculate_quorum(int allow_decrease, int max_expected, unsigned int *
 }
 
 /* Recalculate cluster quorum, set quorate and notify changes */
-static void recalculate_quorum(int allow_decrease)
+static void recalculate_quorum(int allow_decrease, int by_current_nodes)
 {
        unsigned int total_votes;
+       int cluster_members = 0;
+       struct list_head *nodelist;
+       struct cluster_node *node;
 
        ENTER();
-       quorum = calculate_quorum(allow_decrease, 0, &total_votes);
+
+       if (by_current_nodes) {
+               list_iterate(nodelist, &cluster_members_list) {
+                       node = list_entry(nodelist, struct cluster_node, list);
+
+                       if (node->state == NODESTATE_MEMBER) {
+                               cluster_members++;
+                       }
+               }
+       }
+
+       quorum = calculate_quorum(allow_decrease, cluster_members, &total_votes);
        set_quorate(total_votes);
        send_quorum_notification(NULL, 0L);
        LEAVE();
@@ -881,7 +895,7 @@ static void quorum_confchg_fn (
                                node->flags |= NODE_FLAGS_BEENDOWN;
                        }
                }
-               recalculate_quorum(leaving);
+               recalculate_quorum(leaving, leaving);
        }
 
        if (member_list_entries) {
@@ -1031,7 +1045,7 @@ static void message_handler_req_exec_quorum_nodeinfo (
        node->flags &= ~NODE_FLAGS_BEENDOWN;
 
        if (new_node || old_votes != node->votes || old_expected != node->expected_votes || old_state != node->state)
-               recalculate_quorum(0);
+               recalculate_quorum(0, 0);
        LEAVE();
 }
 
@@ -1075,16 +1089,19 @@ static void message_handler_req_exec_quorum_reconfigure (
                                node->expected_votes = req_exec_quorum_reconfigure->value;
                        }
                }
-               recalculate_quorum(1);  /* Allow decrease */
+               recalculate_quorum(1, 0);  /* Allow decrease */
                break;
 
        case RECONFIG_PARAM_NODE_VOTES:
                node->votes = req_exec_quorum_reconfigure->value;
-               recalculate_quorum(1);  /* Allow decrease */
+               recalculate_quorum(1, 0);  /* Allow decrease */
                break;
 
        case RECONFIG_PARAM_LEAVING:
-               node->state = NODESTATE_LEAVING;
+               if (req_exec_quorum_reconfigure->value == 1 && node->state == NODESTATE_MEMBER)
+                       node->state = NODESTATE_LEAVING;
+               if (req_exec_quorum_reconfigure->value == 0 && node->state == NODESTATE_LEAVING)
+                       node->state = NODESTATE_MEMBER;
                break;
        }
 }
@@ -1102,6 +1119,22 @@ static int quorum_lib_init_fn (void *conn)
        return (0);
 }
 
+/*
+ * Someone called votequorum_leave AGES ago!
+ * Assume they forgot to shut down the node.
+ */
+static void leaving_timer_fn(void *arg)
+{
+       ENTER();
+
+       if (us->state == NODESTATE_LEAVING)
+               us->state = NODESTATE_MEMBER;
+
+       /* Tell everyone else we made a mistake */
+       quorum_exec_send_reconfigure(RECONFIG_PARAM_LEAVING, us->node_id, 0);
+       LEAVE();
+}
+
 /* Message from the library */
 static void message_handler_req_lib_votequorum_getinfo (void *conn, void *message)
 {
@@ -1259,7 +1292,16 @@ static void message_handler_req_lib_votequorum_leaving (void *conn, void *messag
 
        ENTER();
 
-       quorum_exec_send_reconfigure(RECONFIG_PARAM_LEAVING, us->node_id, 0);
+       quorum_exec_send_reconfigure(RECONFIG_PARAM_LEAVING, us->node_id, 1);
+
+       /*
+        * If we don't shut down in a sensible amount of time then cancel the
+        * leave status.
+        */
+       if (leaving_timeout)
+               corosync_api->timer_add_duration((unsigned long long)leaving_timeout*1000000, NULL,
+                                                leaving_timer_fn, &leaving_timer);
+
 
        /* send status */
        res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status);
@@ -1280,7 +1322,7 @@ static void quorum_device_timer_fn(void *arg)
        if (quorum_device->last_hello.tv_sec + quorumdev_poll/1000 < now.tv_sec) {
                quorum_device->state = NODESTATE_DEAD;
                log_printf(LOG_INFO, "lost contact with quorum device\n");
-               recalculate_quorum(0);
+               recalculate_quorum(0, 0);
        }
        else {
                corosync_api->timer_add_duration((unsigned long long)quorumdev_poll*1000000, quorum_device,
@@ -1330,7 +1372,7 @@ static void message_handler_req_lib_votequorum_qdisk_unregister (void *conn, voi
                quorum_device = NULL;
                list_del(&node->list);
                free(node);
-               recalculate_quorum(0);
+               recalculate_quorum(0, 0);
        }
        else {
                error = CS_ERR_NOT_EXIST;
@@ -1357,7 +1399,7 @@ static void message_handler_req_lib_votequorum_qdisk_poll (void *conn, void *mes
                        gettimeofday(&quorum_device->last_hello, NULL);
                        if (quorum_device->state == NODESTATE_DEAD) {
                                quorum_device->state = NODESTATE_MEMBER;
-                               recalculate_quorum(0);
+                               recalculate_quorum(0, 0);
 
                                corosync_api->timer_add_duration((unsigned long long)quorumdev_poll*1000000, quorum_device,
                                                                 quorum_device_timer_fn, &quorum_device_timer);
@@ -1366,7 +1408,7 @@ static void message_handler_req_lib_votequorum_qdisk_poll (void *conn, void *mes
                else {
                        if (quorum_device->state == NODESTATE_MEMBER) {
                                quorum_device->state = NODESTATE_DEAD;
-                               recalculate_quorum(0);
+                               recalculate_quorum(0, 0);
                                corosync_api->timer_delete(quorum_device_timer);
                        }
                }