[mirror_corosync-qdevice.git] / qdevices / qdevice-net-heuristics.c

/*
 * Copyright (c) 2017-2020 Red Hat, Inc.
 *
 * All rights reserved.
 *
 * Author: Jan Friesse (jfriesse@redhat.com)
 *
 * This software licensed under BSD license, the text of which follows:
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 * - Neither the name of the Red Hat, Inc. nor the names of its
 *   contributors may be used to endorse or promote products derived from this
 *   software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "log.h"
#include "qdevice-net-algorithm.h"
#include "qdevice-net-cast-vote-timer.h"
#include "qdevice-net-heuristics.h"
#include "qdevice-net-send.h"
#include "qdevice-net-votequorum.h"

enum tlv_heuristics
qdevice_net_heuristics_exec_result_to_tlv(enum qdevice_heuristics_exec_result exec_result)
{
	enum tlv_heuristics res;

	switch (exec_result) {
	case QDEVICE_HEURISTICS_EXEC_RESULT_DISABLED: res = TLV_HEURISTICS_UNDEFINED; break;
	case QDEVICE_HEURISTICS_EXEC_RESULT_PASS: res = TLV_HEURISTICS_PASS; break;
	case QDEVICE_HEURISTICS_EXEC_RESULT_FAIL: res = TLV_HEURISTICS_FAIL; break;
	default:
		log(LOG_ERR, "qdevice_net_heuristics_exec_result_to_tlv: Unhandled "
		    "heuristics exec result %s",
		    qdevice_heuristics_exec_result_to_str(exec_result));
		exit(EXIT_FAILURE);
		break;
	}

	return (res);
}

static int
qdevice_net_regular_heuristics_exec_result_callback(uint32_t seq_number,
    enum qdevice_heuristics_exec_result exec_result, void *user_data1, void *user_data2)
{
	struct qdevice_heuristics_instance *heuristics_instance;
	struct qdevice_instance *instance;
	struct qdevice_net_instance *net_instance;
	int send_msg;
	enum tlv_vote vote;
	enum tlv_heuristics heuristics;

	instance = (struct qdevice_instance *)user_data1;
	heuristics_instance = &instance->heuristics_instance;
	net_instance = instance->model_data;

	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	    qdevice_net_regular_heuristics_exec_result_callback, 0) != 0) {
		log(LOG_ERR, "Can't deactivate net regular heuristics exec callback notifier");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
		net_instance->schedule_disconnect = 1;

		return (0);
	}

	heuristics = qdevice_net_heuristics_exec_result_to_tlv(exec_result);

	if (exec_result == QDEVICE_HEURISTICS_EXEC_RESULT_DISABLED) {
		/*
		 * Can happen when user disables heuristics during runtime
		 */
		return (0);
	}

	if (net_instance->latest_heuristics_result != heuristics) {
		log(heuristics == TLV_HEURISTICS_PASS ? LOG_NOTICE : LOG_ERR,
		    "Heuristics result changed from %s to %s",
		    tlv_heuristics_to_str(net_instance->latest_heuristics_result),
		    tlv_heuristics_to_str(heuristics));

		if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
			/*
			 * Not connected to qnetd
			 */
			send_msg = 0;
		} else {
			send_msg = 1;
		}

		vote = TLV_VOTE_NO_CHANGE;

		if (qdevice_net_algorithm_heuristics_change(net_instance, &heuristics, &send_msg,
		    &vote) == -1) {
			log(LOG_ERR, "Algorithm returned error. Disconnecting.");

			net_instance->disconnect_reason =
			    QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_ERR;
			net_instance->schedule_disconnect = 1;

			return (0);
		} else {
			log(LOG_DEBUG, "Algorithm decided to %s message with heuristics result "
			    "%s and result vote is %s", (send_msg ? "send" : "not send"),
			    tlv_heuristics_to_str(heuristics), tlv_vote_to_str(vote));
		}

		if (send_msg) {
			if (heuristics == TLV_HEURISTICS_UNDEFINED) {
				log(LOG_ERR, "Inconsistent algorithm result. "
				    "It's not possible to send message with undefined heuristics. "
				    "Disconnecting.");

				net_instance->disconnect_reason =
				    QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_ERR;
				net_instance->schedule_disconnect = 1;

				return (0);
			}

			if (!net_instance->server_supports_heuristics) {
				log(LOG_ERR, "Server doesn't support heuristics. "
				    "Disconnecting.");

				net_instance->disconnect_reason =
				    QDEVICE_NET_DISCONNECT_REASON_SERVER_DOESNT_SUPPORT_REQUIRED_OPT;
				net_instance->schedule_disconnect = 1;

				return (0);
			}

			if (qdevice_net_send_heuristics_change(net_instance, heuristics) != 0) {
				net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
				net_instance->schedule_disconnect = 1;

				return (0);
			}
		}

		if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
			log(LOG_CRIT, "qdevice_net_heuristics_exec_result_callback "
			    "Can't update cast vote timer");

			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
			net_instance->schedule_disconnect = 1;

			return (0);
		}
	}

	net_instance->latest_regular_heuristics_result = heuristics;
	net_instance->latest_heuristics_result = heuristics;

	if (qdevice_net_heuristics_schedule_timer(net_instance) != 0) {
		return (0);
	}

	return (0);
}

static int
qdevice_net_connect_heuristics_exec_result_callback(uint32_t seq_number,
    enum qdevice_heuristics_exec_result exec_result, void *user_data1, void *user_data2)
{
	struct qdevice_heuristics_instance *heuristics_instance;
	struct qdevice_instance *instance;
	struct qdevice_net_instance *net_instance;
	enum tlv_vote vote;
	enum tlv_heuristics heuristics;
	int send_config_node_list;
	int send_membership_node_list;
	int send_quorum_node_list;
	struct tlv_ring_id tlv_rid;
	enum tlv_quorate quorate;

	instance = (struct qdevice_instance *)user_data1;
	heuristics_instance = &instance->heuristics_instance;
	net_instance = instance->model_data;

	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	    qdevice_net_connect_heuristics_exec_result_callback, 0) != 0) {
		log(LOG_ERR, "Can't deactivate net connect heuristics exec callback notifier");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
		net_instance->schedule_disconnect = 1;

		return (0);
	}

	if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_INIT_REPLY) {
		/*
		 * Not connected to qnetd -> heuristics will be called again on new connect
		 */
		log(LOG_DEBUG, "Received unexpected net connect heuristics in state %u",
		    net_instance->state);
		return (0);
	}

	heuristics = qdevice_net_heuristics_exec_result_to_tlv(exec_result);

	send_config_node_list = 1;
	send_membership_node_list = 1;
	send_quorum_node_list = 1;
	vote = TLV_VOTE_WAIT_FOR_REPLY;

	if (qdevice_net_algorithm_connected(net_instance, &heuristics, &send_config_node_list,
	    &send_membership_node_list, &send_quorum_node_list, &vote) != 0) {
		log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_CONNECTED_ERR;
		return (0);
	} else {
		log(LOG_DEBUG, "Algorithm decided to %s config node list, %s membership "
		    "node list, %s quorum node list, heuristics is %s and result vote is %s",
		    (send_config_node_list ? "send" : "not send"),
		    (send_membership_node_list ? "send" : "not send"),
		    (send_quorum_node_list ? "send" : "not send"),
		    tlv_heuristics_to_str(heuristics),
		    tlv_vote_to_str(vote));
	}

	/*
	 * Now we can finally really send node list, votequorum node list and update timer
	 */
	if (send_config_node_list) {
		if (qdevice_net_send_config_node_list(net_instance,
		    &instance->config_node_list,
		    instance->config_node_list_version_set,
		    instance->config_node_list_version, 1) != 0) {
			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
			return (0);
		}
	}

	if (send_membership_node_list) {
		qdevice_net_votequorum_ring_id_to_tlv(&tlv_rid,
		    &instance->vq_node_list_ring_id);

		if (qdevice_net_send_membership_node_list(net_instance, &tlv_rid,
		    instance->vq_node_list_entries,
		    instance->vq_node_list,
		    heuristics) != 0) {
			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
			return (0);
		}
	}

	if (send_quorum_node_list) {
		quorate = (instance->vq_quorum_quorate ?
		    TLV_QUORATE_QUORATE : TLV_QUORATE_INQUORATE);

		if (qdevice_net_send_quorum_node_list(net_instance,
		    quorate,
		    instance->vq_quorum_node_list_entries,
		    instance->vq_quorum_node_list) != 0) {
			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
			return (0);
		}
	}

	if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
		log(LOG_CRIT, "qdevice_net_msg_received_set_option_reply fatal error. "
		    " Can't update cast vote timer vote");
		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
	}

	net_instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS;
	net_instance->connected_since_time = time(NULL);

	net_instance->latest_connect_heuristics_result = heuristics;
	net_instance->latest_heuristics_result = heuristics;

	return (0);
}

static int
qdevice_net_heuristics_timer_callback(void *data1, void *data2)
{
	struct qdevice_net_instance *net_instance;
	struct qdevice_heuristics_instance *heuristics_instance;

	net_instance = (struct qdevice_net_instance *)data1;
	heuristics_instance = &net_instance->qdevice_instance_ptr->heuristics_instance;

	if (qdevice_heuristics_waiting_for_result(heuristics_instance)) {
		log(LOG_DEBUG, "Not executing regular heuristics because other heuristics is already running.");

		return (1);
	}

	net_instance->regular_heuristics_timer = NULL;

	log(LOG_DEBUG, "Executing regular heuristics.");

	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	    qdevice_net_regular_heuristics_exec_result_callback, 1) != 0) {
		log(LOG_ERR, "Can't activate net regular heuristics exec callback notifier");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
		net_instance->schedule_disconnect = 1;

		return (0);
	}

	if (qdevice_heuristics_exec(heuristics_instance,
	    net_instance->qdevice_instance_ptr->sync_in_progress) != 0) {
		log(LOG_ERR, "Can't execute regular heuristics.");

		net_instance->schedule_disconnect = 1;
		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_HEURISTICS;

		return (0);
	}

	/*
	 * Do not schedule this callback again. It's going to be scheduled in the
	 * qdevice_net_heuristics_exec_result_callback
	 */
	return (0);
}

int
qdevice_net_heuristics_stop_timer(struct qdevice_net_instance *net_instance)
{
	struct qdevice_instance *instance;
	struct qdevice_heuristics_instance *heuristics_instance;

	instance = net_instance->qdevice_instance_ptr;
	heuristics_instance = &instance->heuristics_instance;

	if (net_instance->regular_heuristics_timer != NULL) {
		log(LOG_DEBUG, "Regular heuristics timer stopped");

		timer_list_delete(&net_instance->main_timer_list, net_instance->regular_heuristics_timer);
		net_instance->regular_heuristics_timer = NULL;

		if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
		    qdevice_net_regular_heuristics_exec_result_callback, 0) != 0) {
			log(LOG_ERR, "Can't deactivate net regular heuristics exec callback notifier");

			net_instance->disconnect_reason =
			    QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
			net_instance->schedule_disconnect = 1;
			return (-1);
		}
	}

	return (0);
}

int
qdevice_net_heuristics_schedule_timer(struct qdevice_net_instance *net_instance)
{
	uint32_t interval;
	struct qdevice_instance *instance;
	struct qdevice_heuristics_instance *heuristics_instance;

	instance = net_instance->qdevice_instance_ptr;
	heuristics_instance = &instance->heuristics_instance;

        if (heuristics_instance->mode != QDEVICE_HEURISTICS_MODE_ENABLED) {
		log(LOG_DEBUG, "Not scheduling heuristics timer because mode is not enabled");

		if (qdevice_net_heuristics_stop_timer(net_instance) != 0) {
			return (-1);
		}

		return (0);
        }

	if (net_instance->regular_heuristics_timer != NULL) {
		log(LOG_DEBUG, "Not scheduling heuristics timer because it is already scheduled");

		return (0);
	}

	interval = heuristics_instance->interval;

	log(LOG_DEBUG, "Scheduling next regular heuristics in %"PRIu32"ms", interval);

	net_instance->regular_heuristics_timer = timer_list_add(&net_instance->main_timer_list,
		interval,
		qdevice_net_heuristics_timer_callback,
	        (void *)net_instance, NULL);

	if (net_instance->regular_heuristics_timer == NULL) {
		log(LOG_ERR, "Can't schedule regular heuristics.");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_HEURISTICS_TIMER;
		net_instance->schedule_disconnect = 1;
		return (-1);
	}

	return (0);
}

int
qdevice_net_heuristics_init(struct qdevice_net_instance *net_instance)
{

	if (qdevice_heuristics_result_notifier_list_add(
	    &net_instance->qdevice_instance_ptr->heuristics_instance.exec_result_notifier_list,
	    qdevice_net_regular_heuristics_exec_result_callback,
	    net_instance->qdevice_instance_ptr, NULL) == NULL) {
		log(LOG_ERR, "Can't add net regular heuristics exec callback into notifier");

		return (-1);
	}

	if (qdevice_heuristics_result_notifier_list_add(
	    &net_instance->qdevice_instance_ptr->heuristics_instance.exec_result_notifier_list,
	    qdevice_net_connect_heuristics_exec_result_callback,
	    net_instance->qdevice_instance_ptr, NULL) == NULL) {
		log(LOG_ERR, "Can't add net connect heuristics exec callback into notifier");

		return (-1);
	}

	return (0);
}

int
qdevice_net_heuristics_exec_after_connect(struct qdevice_net_instance *net_instance)
{
	struct qdevice_instance *instance;
	struct qdevice_heuristics_instance *heuristics_instance;

	instance = net_instance->qdevice_instance_ptr;
	heuristics_instance = &instance->heuristics_instance;

	log(LOG_DEBUG, "Executing after-connect heuristics.");

	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	    qdevice_net_connect_heuristics_exec_result_callback, 1) != 0) {
		log(LOG_ERR, "Can't activate net connect heuristics exec callback notifier");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
		net_instance->schedule_disconnect = 1;

		return (-1);
	}

	if (qdevice_heuristics_exec(heuristics_instance,
	    instance->sync_in_progress) != 0) {
		log(LOG_ERR, "Can't execute connect heuristics.");

		net_instance->schedule_disconnect = 1;
		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_HEURISTICS;

		return (-1);
	}

	return (0);
}
Commit	Line	Data
9a1955a7	1	/*
406b689d	2	* Copyright (c) 2017-2020 Red Hat, Inc.
9a1955a7 JF	3	*
	4	* All rights reserved.
	5	*
	6	* Author: Jan Friesse (jfriesse@redhat.com)
	7	*
	8	* This software licensed under BSD license, the text of which follows:
	9	*
	10	* Redistribution and use in source and binary forms, with or without
	11	* modification, are permitted provided that the following conditions are met:
	12	*
	13	* - Redistributions of source code must retain the above copyright notice,
	14	* this list of conditions and the following disclaimer.
	15	* - Redistributions in binary form must reproduce the above copyright notice,
	16	* this list of conditions and the following disclaimer in the documentation
	17	* and/or other materials provided with the distribution.
	18	* - Neither the name of the Red Hat, Inc. nor the names of its
	19	* contributors may be used to endorse or promote products derived from this
	20	* software without specific prior written permission.
	21	*
	22	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	23	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	24	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	25	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
	26	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	27	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	28	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	29	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	30	* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	31	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
	32	* THE POSSIBILITY OF SUCH DAMAGE.
	33	*/
	34
313d42d1	35	#include "log.h"
9a1955a7 JF	36	#include "qdevice-net-algorithm.h"
	37	#include "qdevice-net-cast-vote-timer.h"
	38	#include "qdevice-net-heuristics.h"
	39	#include "qdevice-net-send.h"
	40	#include "qdevice-net-votequorum.h"
	41
	42	enum tlv_heuristics
	43	qdevice_net_heuristics_exec_result_to_tlv(enum qdevice_heuristics_exec_result exec_result)
	44	{
	45	enum tlv_heuristics res;
	46
	47	switch (exec_result) {
	48	case QDEVICE_HEURISTICS_EXEC_RESULT_DISABLED: res = TLV_HEURISTICS_UNDEFINED; break;
	49	case QDEVICE_HEURISTICS_EXEC_RESULT_PASS: res = TLV_HEURISTICS_PASS; break;
	50	case QDEVICE_HEURISTICS_EXEC_RESULT_FAIL: res = TLV_HEURISTICS_FAIL; break;
	51	default:
c8d19612	52	log(LOG_ERR, "qdevice_net_heuristics_exec_result_to_tlv: Unhandled "
9a1955a7 JF	53	"heuristics exec result %s",
9a1955a7 JF	54	qdevice_heuristics_exec_result_to_str(exec_result));
406b689d	55	exit(EXIT_FAILURE);
9a1955a7 JF	56	break;
	57	}
	58
	59	return (res);
	60	}
	61
	62	static int
ae7d6029 JF	63	qdevice_net_regular_heuristics_exec_result_callback(uint32_t seq_number,
ae7d6029 JF	64	enum qdevice_heuristics_exec_result exec_result, void user_data1, void user_data2)
9a1955a7 JF	65	{
	66	struct qdevice_heuristics_instance *heuristics_instance;
	67	struct qdevice_instance *instance;
	68	struct qdevice_net_instance *net_instance;
	69	int send_msg;
	70	enum tlv_vote vote;
	71	enum tlv_heuristics heuristics;
	72
ae7d6029 JF	73	instance = (struct qdevice_instance *)user_data1;
ae7d6029 JF	74	heuristics_instance = &instance->heuristics_instance;
9a1955a7 JF	75	net_instance = instance->model_data;
	76
	77	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	78	qdevice_net_regular_heuristics_exec_result_callback, 0) != 0) {
c8d19612	79	log(LOG_ERR, "Can't deactivate net regular heuristics exec callback notifier");
9a1955a7 JF	80
	81	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
	82	net_instance->schedule_disconnect = 1;
	83
	84	return (0);
	85	}
	86
	87	heuristics = qdevice_net_heuristics_exec_result_to_tlv(exec_result);
	88
	89	if (exec_result == QDEVICE_HEURISTICS_EXEC_RESULT_DISABLED) {
	90	/*
	91	* Can happen when user disables heuristics during runtime
	92	*/
	93	return (0);
	94	}
	95
	96	if (net_instance->latest_heuristics_result != heuristics) {
5cb2ff57	97	log(heuristics == TLV_HEURISTICS_PASS ? LOG_NOTICE : LOG_ERR,
5cb2ff57	98	"Heuristics result changed from %s to %s",
9a1955a7 JF	99	tlv_heuristics_to_str(net_instance->latest_heuristics_result),
	100	tlv_heuristics_to_str(heuristics));
	101
	102	if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
	103	/*
	104	* Not connected to qnetd
	105	*/
	106	send_msg = 0;
	107	} else {
	108	send_msg = 1;
	109	}
	110
	111	vote = TLV_VOTE_NO_CHANGE;
	112
	113	if (qdevice_net_algorithm_heuristics_change(net_instance, &heuristics, &send_msg,
	114	&vote) == -1) {
c8d19612	115	log(LOG_ERR, "Algorithm returned error. Disconnecting.");
9a1955a7 JF	116
	117	net_instance->disconnect_reason =
	118	QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_ERR;
	119	net_instance->schedule_disconnect = 1;
	120
	121	return (0);
	122	} else {
c8d19612	123	log(LOG_DEBUG, "Algorithm decided to %s message with heuristics result "
9a1955a7 JF	124	"%s and result vote is %s", (send_msg ? "send" : "not send"),
	125	tlv_heuristics_to_str(heuristics), tlv_vote_to_str(vote));
	126	}
	127
	128	if (send_msg) {
	129	if (heuristics == TLV_HEURISTICS_UNDEFINED) {
c8d19612	130	log(LOG_ERR, "Inconsistent algorithm result. "
9a1955a7 JF	131	"It's not possible to send message with undefined heuristics. "
	132	"Disconnecting.");
	133
	134	net_instance->disconnect_reason =
	135	QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_ERR;
	136	net_instance->schedule_disconnect = 1;
	137
	138	return (0);
	139	}
	140
	141	if (!net_instance->server_supports_heuristics) {
c8d19612	142	log(LOG_ERR, "Server doesn't support heuristics. "
9a1955a7 JF	143	"Disconnecting.");
	144
	145	net_instance->disconnect_reason =
	146	QDEVICE_NET_DISCONNECT_REASON_SERVER_DOESNT_SUPPORT_REQUIRED_OPT;
	147	net_instance->schedule_disconnect = 1;
	148
	149	return (0);
	150	}
	151
	152	if (qdevice_net_send_heuristics_change(net_instance, heuristics) != 0) {
	153	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
	154	net_instance->schedule_disconnect = 1;
	155
	156	return (0);
	157	}
	158	}
	159
	160	if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
c8d19612	161	log(LOG_CRIT, "qdevice_net_heuristics_exec_result_callback "
9a1955a7 JF	162	"Can't update cast vote timer");
	163
	164	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
	165	net_instance->schedule_disconnect = 1;
	166
	167	return (0);
	168	}
	169	}
	170
	171	net_instance->latest_regular_heuristics_result = heuristics;
	172	net_instance->latest_heuristics_result = heuristics;
	173
	174	if (qdevice_net_heuristics_schedule_timer(net_instance) != 0) {
	175	return (0);
	176	}
	177
	178	return (0);
	179	}
	180
	181	static int
ae7d6029 JF	182	qdevice_net_connect_heuristics_exec_result_callback(uint32_t seq_number,
ae7d6029 JF	183	enum qdevice_heuristics_exec_result exec_result, void user_data1, void user_data2)
9a1955a7 JF	184	{
	185	struct qdevice_heuristics_instance *heuristics_instance;
	186	struct qdevice_instance *instance;
	187	struct qdevice_net_instance *net_instance;
	188	enum tlv_vote vote;
	189	enum tlv_heuristics heuristics;
	190	int send_config_node_list;
	191	int send_membership_node_list;
	192	int send_quorum_node_list;
	193	struct tlv_ring_id tlv_rid;
	194	enum tlv_quorate quorate;
	195
ae7d6029 JF	196	instance = (struct qdevice_instance *)user_data1;
ae7d6029 JF	197	heuristics_instance = &instance->heuristics_instance;
9a1955a7 JF	198	net_instance = instance->model_data;
9a1955a7 JF	199
9a1955a7 JF	200	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
9a1955a7 JF	201	qdevice_net_connect_heuristics_exec_result_callback, 0) != 0) {
c8d19612	202	log(LOG_ERR, "Can't deactivate net connect heuristics exec callback notifier");
9a1955a7 JF	203
	204	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
	205	net_instance->schedule_disconnect = 1;
	206
	207	return (0);
	208	}
	209
6bf5f6c0	210	if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_INIT_REPLY) {
8dbf1bc8 JF	211	/*
	212	* Not connected to qnetd -> heuristics will be called again on new connect
	213	*/
6bf5f6c0 JF	214	log(LOG_DEBUG, "Received unexpected net connect heuristics in state %u",
6bf5f6c0 JF	215	net_instance->state);
8dbf1bc8 JF	216	return (0);
	217	}
	218
9a1955a7 JF	219	heuristics = qdevice_net_heuristics_exec_result_to_tlv(exec_result);
	220
	221	send_config_node_list = 1;
	222	send_membership_node_list = 1;
	223	send_quorum_node_list = 1;
	224	vote = TLV_VOTE_WAIT_FOR_REPLY;
	225
	226	if (qdevice_net_algorithm_connected(net_instance, &heuristics, &send_config_node_list,
	227	&send_membership_node_list, &send_quorum_node_list, &vote) != 0) {
c8d19612	228	log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
9a1955a7 JF	229	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_CONNECTED_ERR;
	230	return (0);
	231	} else {
c8d19612	232	log(LOG_DEBUG, "Algorithm decided to %s config node list, %s membership "
9a1955a7 JF	233	"node list, %s quorum node list, heuristics is %s and result vote is %s",
	234	(send_config_node_list ? "send" : "not send"),
	235	(send_membership_node_list ? "send" : "not send"),
	236	(send_quorum_node_list ? "send" : "not send"),
	237	tlv_heuristics_to_str(heuristics),
	238	tlv_vote_to_str(vote));
	239	}
	240
	241	/*
	242	* Now we can finally really send node list, votequorum node list and update timer
	243	*/
	244	if (send_config_node_list) {
	245	if (qdevice_net_send_config_node_list(net_instance,
	246	&instance->config_node_list,
	247	instance->config_node_list_version_set,
	248	instance->config_node_list_version, 1) != 0) {
	249	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
	250	return (0);
	251	}
	252	}
	253
	254	if (send_membership_node_list) {
	255	qdevice_net_votequorum_ring_id_to_tlv(&tlv_rid,
	256	&instance->vq_node_list_ring_id);
	257
	258	if (qdevice_net_send_membership_node_list(net_instance, &tlv_rid,
	259	instance->vq_node_list_entries,
	260	instance->vq_node_list,
	261	heuristics) != 0) {
	262	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
	263	return (0);
	264	}
	265	}
	266
	267	if (send_quorum_node_list) {
	268	quorate = (instance->vq_quorum_quorate ?
	269	TLV_QUORATE_QUORATE : TLV_QUORATE_INQUORATE);
	270
	271	if (qdevice_net_send_quorum_node_list(net_instance,
	272	quorate,
	273	instance->vq_quorum_node_list_entries,
	274	instance->vq_quorum_node_list) != 0) {
	275	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
	276	return (0);
	277	}
	278	}
	279
	280	if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
c8d19612	281	log(LOG_CRIT, "qdevice_net_msg_received_set_option_reply fatal error. "
9a1955a7 JF	282	" Can't update cast vote timer vote");
	283	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
	284	}
	285
	286	net_instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS;
	287	net_instance->connected_since_time = time(NULL);
	288
	289	net_instance->latest_connect_heuristics_result = heuristics;
	290	net_instance->latest_heuristics_result = heuristics;
	291
	292	return (0);
	293	}
	294
	295	static int
	296	qdevice_net_heuristics_timer_callback(void data1, void data2)
	297	{
	298	struct qdevice_net_instance *net_instance;
	299	struct qdevice_heuristics_instance *heuristics_instance;
	300
	301	net_instance = (struct qdevice_net_instance *)data1;
	302	heuristics_instance = &net_instance->qdevice_instance_ptr->heuristics_instance;
	303
	304	if (qdevice_heuristics_waiting_for_result(heuristics_instance)) {
c8d19612	305	log(LOG_DEBUG, "Not executing regular heuristics because other heuristics is already running.");
9a1955a7 JF	306
	307	return (1);
	308	}
	309
	310	net_instance->regular_heuristics_timer = NULL;
	311
c8d19612	312	log(LOG_DEBUG, "Executing regular heuristics.");
9a1955a7 JF	313
	314	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	315	qdevice_net_regular_heuristics_exec_result_callback, 1) != 0) {
c8d19612	316	log(LOG_ERR, "Can't activate net regular heuristics exec callback notifier");
9a1955a7 JF	317
	318	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
	319	net_instance->schedule_disconnect = 1;
	320
	321	return (0);
	322	}
	323
	324	if (qdevice_heuristics_exec(heuristics_instance,
	325	net_instance->qdevice_instance_ptr->sync_in_progress) != 0) {
c8d19612	326	log(LOG_ERR, "Can't execute regular heuristics.");
9a1955a7 JF	327
	328	net_instance->schedule_disconnect = 1;
	329	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_HEURISTICS;
	330
	331	return (0);
	332	}
	333
	334	/*
	335	* Do not schedule this callback again. It's going to be scheduled in the
	336	* qdevice_net_heuristics_exec_result_callback
	337	*/
	338	return (0);
	339	}
	340
	341	int
	342	qdevice_net_heuristics_stop_timer(struct qdevice_net_instance *net_instance)
	343	{
	344	struct qdevice_instance *instance;
	345	struct qdevice_heuristics_instance *heuristics_instance;
	346
	347	instance = net_instance->qdevice_instance_ptr;
	348	heuristics_instance = &instance->heuristics_instance;
	349
	350	if (net_instance->regular_heuristics_timer != NULL) {
c8d19612	351	log(LOG_DEBUG, "Regular heuristics timer stopped");
9a1955a7 JF	352
	353	timer_list_delete(&net_instance->main_timer_list, net_instance->regular_heuristics_timer);
	354	net_instance->regular_heuristics_timer = NULL;
	355
	356	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	357	qdevice_net_regular_heuristics_exec_result_callback, 0) != 0) {
c8d19612	358	log(LOG_ERR, "Can't deactivate net regular heuristics exec callback notifier");
9a1955a7 JF	359
	360	net_instance->disconnect_reason =
	361	QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
	362	net_instance->schedule_disconnect = 1;
	363	return (-1);
	364	}
	365	}
	366
	367	return (0);
	368	}
	369
	370	int
	371	qdevice_net_heuristics_schedule_timer(struct qdevice_net_instance *net_instance)
	372	{
	373	uint32_t interval;
	374	struct qdevice_instance *instance;
	375	struct qdevice_heuristics_instance *heuristics_instance;
	376
	377	instance = net_instance->qdevice_instance_ptr;
	378	heuristics_instance = &instance->heuristics_instance;
	379
	380	if (heuristics_instance->mode != QDEVICE_HEURISTICS_MODE_ENABLED) {
c8d19612	381	log(LOG_DEBUG, "Not scheduling heuristics timer because mode is not enabled");
9a1955a7 JF	382
	383	if (qdevice_net_heuristics_stop_timer(net_instance) != 0) {
	384	return (-1);
	385	}
	386
	387	return (0);
	388	}
	389
	390	if (net_instance->regular_heuristics_timer != NULL) {
c8d19612	391	log(LOG_DEBUG, "Not scheduling heuristics timer because it is already scheduled");
9a1955a7 JF	392
	393	return (0);
	394	}
	395
	396	interval = heuristics_instance->interval;
	397
c8d19612	398	log(LOG_DEBUG, "Scheduling next regular heuristics in %"PRIu32"ms", interval);
9a1955a7 JF	399
	400	net_instance->regular_heuristics_timer = timer_list_add(&net_instance->main_timer_list,
	401	interval,
	402	qdevice_net_heuristics_timer_callback,
	403	(void *)net_instance, NULL);
	404
	405	if (net_instance->regular_heuristics_timer == NULL) {
c8d19612	406	log(LOG_ERR, "Can't schedule regular heuristics.");
9a1955a7 JF	407
	408	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_HEURISTICS_TIMER;
	409	net_instance->schedule_disconnect = 1;
	410	return (-1);
	411	}
	412
	413	return (0);
	414	}
	415
	416	int
	417	qdevice_net_heuristics_init(struct qdevice_net_instance *net_instance)
	418	{
	419
	420	if (qdevice_heuristics_result_notifier_list_add(
	421	&net_instance->qdevice_instance_ptr->heuristics_instance.exec_result_notifier_list,
ae7d6029 JF	422	qdevice_net_regular_heuristics_exec_result_callback,
ae7d6029 JF	423	net_instance->qdevice_instance_ptr, NULL) == NULL) {
c8d19612	424	log(LOG_ERR, "Can't add net regular heuristics exec callback into notifier");
9a1955a7 JF	425
	426	return (-1);
	427	}
	428
	429	if (qdevice_heuristics_result_notifier_list_add(
	430	&net_instance->qdevice_instance_ptr->heuristics_instance.exec_result_notifier_list,
ae7d6029 JF	431	qdevice_net_connect_heuristics_exec_result_callback,
ae7d6029 JF	432	net_instance->qdevice_instance_ptr, NULL) == NULL) {
c8d19612	433	log(LOG_ERR, "Can't add net connect heuristics exec callback into notifier");
9a1955a7 JF	434
	435	return (-1);
	436	}
	437
	438	return (0);
	439	}
	440
	441	int
	442	qdevice_net_heuristics_exec_after_connect(struct qdevice_net_instance *net_instance)
	443	{
	444	struct qdevice_instance *instance;
	445	struct qdevice_heuristics_instance *heuristics_instance;
	446
	447	instance = net_instance->qdevice_instance_ptr;
	448	heuristics_instance = &instance->heuristics_instance;
	449
c8d19612	450	log(LOG_DEBUG, "Executing after-connect heuristics.");
9a1955a7 JF	451
	452	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	453	qdevice_net_connect_heuristics_exec_result_callback, 1) != 0) {
c8d19612	454	log(LOG_ERR, "Can't activate net connect heuristics exec callback notifier");
9a1955a7 JF	455
	456	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
	457	net_instance->schedule_disconnect = 1;
	458
	459	return (-1);
	460	}
	461
	462	if (qdevice_heuristics_exec(heuristics_instance,
	463	instance->sync_in_progress) != 0) {
c8d19612	464	log(LOG_ERR, "Can't execute connect heuristics.");
9a1955a7 JF	465
	466	net_instance->schedule_disconnect = 1;
	467	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_HEURISTICS;
	468
	469	return (-1);
	470	}
	471
	472	return (0);
	473	}