[pve-cluster.git] / data / PVE / API2 / ClusterConfig.pm

package PVE::API2::ClusterConfig;

use strict;
use warnings;

use PVE::Tools;
use PVE::SafeSyslog;
use PVE::RESTHandler;
use PVE::RPCEnvironment;
use PVE::JSONSchema qw(get_standard_option);
use PVE::Cluster;
use PVE::APIClient::LWP;
use PVE::Corosync;

use IO::Socket::UNIX;

use base qw(PVE::RESTHandler);

my $clusterconf = "/etc/pve/corosync.conf";
my $authfile = "/etc/corosync/authkey";
my $local_cluster_lock = "/var/lock/pvecm.lock";

my $nodeid_desc = {
    type => 'integer',
    description => "Node id for this node.",
    minimum => 1,
    optional => 1,
};
PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc);

__PACKAGE__->register_method({
    name => 'index',
    path => '',
    method => 'GET',
    description => "Directory index.",
    permissions => {
	check => ['perm', '/', [ 'Sys.Audit' ]],
    },
    parameters => {
	additionalProperties => 0,
	properties => {},
    },
    returns => {
	type => 'array',
	items => {
	    type => "object",
	    properties => {},
	},
	links => [ { rel => 'child', href => "{name}" } ],
    },
    code => sub {
	my ($param) = @_;

	my $result = [
	    { name => 'nodes' },
	    { name => 'totem' },
	    { name => 'join' },
	    { name => 'qdevice' },
	];

	return $result;
    }});

__PACKAGE__->register_method ({
    name => 'create',
    path => '',
    method => 'POST',
    protected => 1,
    description => "Generate new cluster configuration.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    clustername => {
		description => "The name of the cluster.",
		type => 'string', format => 'pve-node',
		maxLength => 15,
	    },
	    nodeid => get_standard_option('corosync-nodeid'),
	    votes => {
		type => 'integer',
		description => "Number of votes for this node.",
		minimum => 1,
		optional => 1,
	    },
	    link0 => get_standard_option('corosync-link'),
	    link1 => get_standard_option('corosync-link'),
	},
    },
    returns => { type => 'string' },
    code => sub {
	my ($param) = @_;

	-f $clusterconf && die "cluster config '$clusterconf' already exists\n";

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $code = sub {
	    STDOUT->autoflush();
	    PVE::Cluster::setup_sshd_config(1);
	    PVE::Cluster::setup_rootsshconfig();
	    PVE::Cluster::setup_ssh_keys();

	    PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile])
		if !-f $authfile;
	    die "no authentication key available\n" if -f !$authfile;

	    my $nodename = PVE::INotify::nodename();

	    # get the corosync basis config for the new cluster
	    my $config = PVE::Corosync::create_conf($nodename, %$param);

	    print "Writing corosync config to /etc/pve/corosync.conf\n";
	    PVE::Corosync::atomic_write_conf($config);

	    my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
	    PVE::Cluster::ssh_merge_keys();
	    PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
	    PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);

	    print "Restart corosync and cluster filesystem\n";
	    PVE::Tools::run_command('systemctl restart corosync pve-cluster');
	};

	my $worker = sub {
	    PVE::Tools::lock_file($local_cluster_lock, 10, $code);
	    die $@ if $@;
	};

	return $rpcenv->fork_worker('clustercreate', $param->{clustername},  $authuser, $worker);
}});

__PACKAGE__->register_method({
    name => 'nodes',
    path => 'nodes',
    method => 'GET',
    description => "Corosync node list.",
    permissions => {
	check => ['perm', '/', [ 'Sys.Audit' ]],
    },
    parameters => {
	additionalProperties => 0,
	properties => {},
    },
    returns => {
	type => 'array',
	items => {
	    type => "object",
	    properties => {
		node => { type => 'string' },
	    },
	},
	links => [ { rel => 'child', href => "{node}" } ],
    },
    code => sub {
	my ($param) = @_;


	my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
	my $nodelist = PVE::Corosync::nodelist($conf);

	return PVE::RESTHandler::hash_to_array($nodelist, 'node');
    }});

# lock method to ensure local and cluster wide atomicity
# if we're a single node cluster just lock locally, we have no other cluster
# node which we could contend with, else also acquire a cluster wide lock
my $config_change_lock = sub {
    my ($code) = @_;

    PVE::Tools::lock_file($local_cluster_lock, 10, sub {
	PVE::Cluster::cfs_update(1);
	my $members = PVE::Cluster::get_members();
	if (scalar(keys %$members) > 1) {
	    my $res = PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);

	    # cfs_lock_file only sets $@ but lock_file doesn't propagates $@ unless we die here
	    die $@ if defined($@);

	    return $res;
	} else {
	    return $code->();
	}
    });
};

__PACKAGE__->register_method ({
    name => 'addnode',
    path => 'nodes/{node}',
    method => 'POST',
    protected => 1,
    description => "Adds a node to the cluster configuration. This call is for internal use.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    nodeid => get_standard_option('corosync-nodeid'),
	    votes => {
		type => 'integer',
		description => "Number of votes for this node",
		minimum => 0,
		optional => 1,
	    },
	    force => {
		type => 'boolean',
		description => "Do not throw error if node already exists.",
		optional => 1,
	    },
	    link0 => get_standard_option('corosync-link'),
	    link1 => get_standard_option('corosync-link'),
	},
    },
    returns => {
	type => "object",
	properties => {
	    corosync_authkey => {
		type => 'string',
	    },
	    corosync_conf => {
		type => 'string',
	    }
	},
    },
    code => sub {
	my ($param) = @_;

	PVE::Cluster::check_cfs_quorum();

	my $code = sub {
	    my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
	    my $nodelist = PVE::Corosync::nodelist($conf);
	    my $totem_cfg = PVE::Corosync::totem_config($conf);

	    my $name = $param->{node};

	    # ensure we do not reuse an address, that can crash the whole cluster!
	    my $check_duplicate_addr = sub {
		my $link = shift;
		return if !defined($link) || !defined($link->{address});
		my $addr = $link->{address};

		while (my ($k, $v) = each %$nodelist) {
		    next if $k eq $name; # allows re-adding a node if force is set

		    for my $linknumber (0..1) {
			my $id = "ring${linknumber}_addr";
			next if !defined($v->{$id});

			die "corosync: address '$addr' already used on link $id by node '$k'\n"
			    if $v->{$id} eq $addr;
		    }
		}
	    };

	    my $link0 = PVE::Cluster::parse_corosync_link($param->{link0});
	    my $link1 = PVE::Cluster::parse_corosync_link($param->{link1});

	    $check_duplicate_addr->($link0);
	    $check_duplicate_addr->($link1);

	    # FIXME: handle all links (0-7), they're all independent now
	    $link0->{address} //= $name if exists($totem_cfg->{interface}->{0});

	    die "corosync: using 'link1' parameter needs a interface with linknumber '1' configured!\n"
		if $link1 && !defined($totem_cfg->{interface}->{1});

	    die "corosync: totem interface with linknumber 1 configured but 'link1' parameter not defined!\n"
		if defined($totem_cfg->{interface}->{1}) && !defined($link1);

	    if (defined(my $res = $nodelist->{$name})) {
		$param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
		$param->{votes} = $res->{quorum_votes} if !defined($param->{votes});

		if ($res->{quorum_votes} == $param->{votes} &&
		    $res->{nodeid} == $param->{nodeid} && $param->{force}) {
		    print "forcing overwrite of configured node '$name'\n";
		} else {
		    die "can't add existing node '$name'\n";
		}
	    } elsif (!$param->{nodeid}) {
		my $nodeid = 1;

		while(1) {
		    my $found = 0;
		    foreach my $v (values %$nodelist) {
			if ($v->{nodeid} eq $nodeid) {
			    $found = 1;
			    $nodeid++;
			    last;
			}
		    }
		    last if !$found;
		};

		$param->{nodeid} = $nodeid;
	    }

	    $param->{votes} = 1 if !defined($param->{votes});

	    PVE::Cluster::gen_local_dirs($name);

	    eval { PVE::Cluster::ssh_merge_keys(); };
	    warn $@ if $@;

	    $nodelist->{$name} = {
		ring0_addr => $link0->{address},
		nodeid => $param->{nodeid},
		name => $name,
	    };
	    $nodelist->{$name}->{ring1_addr} = $link1->{address} if defined($link1);
	    $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};

	    PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster");

	    PVE::Corosync::update_nodelist($conf, $nodelist);
	};

	$config_change_lock->($code);
	die $@ if $@;

	my $res = {
	    corosync_authkey => PVE::Tools::file_get_contents($authfile),
	    corosync_conf => PVE::Tools::file_get_contents($clusterconf),
	};

	return $res;
    }});


__PACKAGE__->register_method ({
    name => 'delnode',
    path => 'nodes/{node}',
    method => 'DELETE',
    protected => 1,
    description => "Removes a node from the cluster configuration.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	my $local_node = PVE::INotify::nodename();
	die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node;

	PVE::Cluster::check_cfs_quorum();

	my $code = sub {
	    my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
	    my $nodelist = PVE::Corosync::nodelist($conf);

	    my $node;
	    my $nodeid;

	    foreach my $tmp_node (keys %$nodelist) {
		my $d = $nodelist->{$tmp_node};
		my $ring0_addr = $d->{ring0_addr};
		my $ring1_addr = $d->{ring1_addr};
		if (($tmp_node eq $param->{node}) ||
		    (defined($ring0_addr) && ($ring0_addr eq $param->{node})) ||
		    (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
		    $node = $tmp_node;
		    $nodeid = $d->{nodeid};
		    last;
		}
	    }

	    die "Node/IP: $param->{node} is not a known host of the cluster.\n"
		if !defined($node);

	    PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster");

	    delete $nodelist->{$node};

	    PVE::Corosync::update_nodelist($conf, $nodelist);

	    PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid]) if defined($nodeid);
	};

	$config_change_lock->($code);
	die $@ if $@;

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'join_info',
    path => 'join',
    permissions => {
	check => ['perm', '/', [ 'Sys.Audit' ]],
    },
    method => 'GET',
    description => "Get information needed to join this cluster over the connected node.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node', {
		description => "The node for which the joinee gets the nodeinfo. ",
		default => "current connected node",
		optional => 1,
	    }),
	},
    },
    returns => {
	type => 'object',
	additionalProperties => 0,
	properties => {
	    nodelist => {
		type => 'array',
		items => {
		    type => "object",
		    additionalProperties => 1,
		    properties => {
			name => get_standard_option('pve-node'),
			nodeid => get_standard_option('corosync-nodeid'),
			ring0_addr => get_standard_option('corosync-link'),
			quorum_votes => { type => 'integer', minimum => 0 },
			pve_addr => { type => 'string', format => 'ip' },
			pve_fp => get_standard_option('fingerprint-sha256'),
		    },
		},
	    },
	    preferred_node => get_standard_option('pve-node'),
	    totem => { type => 'object' },
	    config_digest => { type => 'string' },
	},
    },
    code => sub {
	my ($param) = @_;

	my $nodename = $param->{node} // PVE::INotify::nodename();

	PVE::Cluster::cfs_update(1);
	my $conf = PVE::Cluster::cfs_read_file('corosync.conf');

	die "node is not in a cluster, no join info available!\n"
	    if !($conf && $conf->{main});

	my $totem_cfg = $conf->{main}->{totem} // {};
	my $nodelist = $conf->{main}->{nodelist}->{node} // {};
	my $corosync_config_digest = $conf->{digest};

	die "unknown node '$nodename'\n" if ! $nodelist->{$nodename};

	foreach my $name (keys %$nodelist) {
	    my $node = $nodelist->{$name};
	    $node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name);
	    $node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name));
	}

	my $res = {
	    nodelist => [ values %$nodelist ],
	    preferred_node => $nodename,
	    totem => $totem_cfg,
	    config_digest => $corosync_config_digest,
	};

	return $res;
    }});

__PACKAGE__->register_method ({
    name => 'join',
    path => 'join',
    method => 'POST',
    protected => 1,
    description => "Joins this node into an existing cluster.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    hostname => {
		type => 'string',
		description => "Hostname (or IP) of an existing cluster member."
	    },
	    nodeid => get_standard_option('corosync-nodeid'),
	    votes => {
		type => 'integer',
		description => "Number of votes for this node",
		minimum => 0,
		optional => 1,
	    },
	    force => {
		type => 'boolean',
		description => "Do not throw error if node already exists.",
		optional => 1,
	    },
	    link0 => get_standard_option('corosync-link', {
		default => "IP resolved by node's hostname",
	    }),
	    link1 => get_standard_option('corosync-link'),
	    fingerprint => get_standard_option('fingerprint-sha256'),
	    password => {
		description => "Superuser (root) password of peer node.",
		type => 'string',
		maxLength => 128,
	    },
	},
    },
    returns => { type => 'string' },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $worker = sub {
	    STDOUT->autoflush();
	    PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::join, $param);
	    die $@ if $@;
	};

	return $rpcenv->fork_worker('clusterjoin', undef,  $authuser, $worker);
    }});


__PACKAGE__->register_method({
    name => 'totem',
    path => 'totem',
    method => 'GET',
    description => "Get corosync totem protocol settings.",
    permissions => {
	check => ['perm', '/', [ 'Sys.Audit' ]],
    },
    parameters => {
	additionalProperties => 0,
	properties => {},
    },
    returns => {
	type => "object",
	properties => {},
    },
    code => sub {
	my ($param) = @_;


	my $conf = PVE::Cluster::cfs_read_file('corosync.conf');

	my $totem_cfg = $conf->{main}->{totem};

	return $totem_cfg;
    }});

__PACKAGE__->register_method ({
    name => 'status',
    path => 'qdevice',
    method => 'GET',
    description => 'Get QDevice status',
    permissions => {
	check => ['perm', '/', [ 'Sys.Audit' ]],
    },
    parameters => {
	additionalProperties => 0,
	properties => {},
    },
    returns => {
	type => "object",
    },
    code => sub {
	my ($param) = @_;

	my $result = {};
	my $socket_path = "/var/run/corosync-qdevice/corosync-qdevice.sock";
	return $result if !-S $socket_path;

	my $qdevice_socket = IO::Socket::UNIX->new(
	    Type => SOCK_STREAM,
	    Peer => $socket_path,
	);

	print $qdevice_socket "status verbose\n";
	my $qdevice_keys = {
	    "Algorithm" => 1,
	    "Echo reply" => 1,
	    "Last poll call" => 1,
	    "Model" => 1,
	    "QNetd host" => 1,
	    "State" => 1,
	    "Tie-breaker" => 1,
	};
	while (my $line = <$qdevice_socket>) {
	    chomp $line;
	    next if $line =~ /^\s/;
	    if ($line =~ /^(.*?)\s*:\s*(.*)$/) {
		$result->{$1} = $2 if $qdevice_keys->{$1};
	    }
	}

	return $result;
    }});
#TODO: possibly add setup and remove methods


1;
Commit	Line	Data
963c06bb DM	1	package PVE::API2::ClusterConfig;
	2
	3	use strict;
	4	use warnings;
b6973a89	5
963c06bb DM	6	use PVE::Tools;
	7	use PVE::SafeSyslog;
	8	use PVE::RESTHandler;
	9	use PVE::RPCEnvironment;
	10	use PVE::JSONSchema qw(get_standard_option);
	11	use PVE::Cluster;
6ed49eb1	12	use PVE::APIClient::LWP;
b6973a89	13	use PVE::Corosync;
963c06bb	14
f0f8ee41 OB	15	use IO::Socket::UNIX;
f0f8ee41 OB	16
963c06bb DM	17	use base qw(PVE::RESTHandler);
963c06bb DM	18
74e09a93 TL	19	my $clusterconf = "/etc/pve/corosync.conf";
74e09a93 TL	20	my $authfile = "/etc/corosync/authkey";
b184a69d	21	my $local_cluster_lock = "/var/lock/pvecm.lock";
74e09a93	22
10c6810e TL	23	my $nodeid_desc = {
	24	type => 'integer',
	25	description => "Node id for this node.",
	26	minimum => 1,
	27	optional => 1,
	28	};
	29	PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc);
	30
963c06bb DM	31	__PACKAGE__->register_method({
	32	name => 'index',
	33	path => '',
	34	method => 'GET',
	35	description => "Directory index.",
fb7c665a EK	36	permissions => {
	37	check => ['perm', '/', [ 'Sys.Audit' ]],
	38	},
963c06bb DM	39	parameters => {
	40	additionalProperties => 0,
	41	properties => {},
	42	},
	43	returns => {
	44	type => 'array',
	45	items => {
	46	type => "object",
	47	properties => {},
	48	},
	49	links => [ { rel => 'child', href => "{name}" } ],
	50	},
	51	code => sub {
	52	my ($param) = @_;
	53
	54	my $result = [
	55	{ name => 'nodes' },
	56	{ name => 'totem' },
6ed49eb1	57	{ name => 'join' },
f0f8ee41	58	{ name => 'qdevice' },
6ed49eb1	59	];
963c06bb DM	60
	61	return $result;
	62	}});
	63
74e09a93 TL	64	__PACKAGE__->register_method ({
	65	name => 'create',
	66	path => '',
	67	method => 'POST',
	68	protected => 1,
	69	description => "Generate new cluster configuration.",
	70	parameters => {
	71	additionalProperties => 0,
	72	properties => {
	73	clustername => {
	74	description => "The name of the cluster.",
	75	type => 'string', format => 'pve-node',
	76	maxLength => 15,
	77	},
10c6810e	78	nodeid => get_standard_option('corosync-nodeid'),
74e09a93 TL	79	votes => {
	80	type => 'integer',
	81	description => "Number of votes for this node.",
	82	minimum => 1,
	83	optional => 1,
	84	},
046173ce TL	85	link0 => get_standard_option('corosync-link'),
046173ce TL	86	link1 => get_standard_option('corosync-link'),
74e09a93 TL	87	},
74e09a93 TL	88	},
333a9bc8	89	returns => { type => 'string' },
74e09a93 TL	90	code => sub {
	91	my ($param) = @_;
	92
	93	-f $clusterconf && die "cluster config '$clusterconf' already exists\n";
	94
333a9bc8 TL	95	my $rpcenv = PVE::RPCEnvironment::get();
	96	my $authuser = $rpcenv->get_user();
	97
b184a69d	98	my $code = sub {
17a4445f	99	STDOUT->autoflush();
333a9bc8 TL	100	PVE::Cluster::setup_sshd_config(1);
	101	PVE::Cluster::setup_rootsshconfig();
	102	PVE::Cluster::setup_ssh_keys();
74e09a93	103
333a9bc8 TL	104	PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile])
	105	if !-f $authfile;
	106	die "no authentication key available\n" if -f !$authfile;
74e09a93	107
333a9bc8	108	my $nodename = PVE::INotify::nodename();
74e09a93	109
333a9bc8 TL	110	# get the corosync basis config for the new cluster
333a9bc8 TL	111	my $config = PVE::Corosync::create_conf($nodename, %$param);
74e09a93	112
333a9bc8 TL	113	print "Writing corosync config to /etc/pve/corosync.conf\n";
333a9bc8 TL	114	PVE::Corosync::atomic_write_conf($config);
74e09a93	115
333a9bc8 TL	116	my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
	117	PVE::Cluster::ssh_merge_keys();
	118	PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
	119	PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
74e09a93	120
333a9bc8 TL	121	print "Restart corosync and cluster filesystem\n";
	122	PVE::Tools::run_command('systemctl restart corosync pve-cluster');
	123	};
74e09a93	124
b184a69d TL	125	my $worker = sub {
	126	PVE::Tools::lock_file($local_cluster_lock, 10, $code);
	127	die $@ if $@;
	128	};
	129
65ee8001	130	return $rpcenv->fork_worker('clustercreate', $param->{clustername}, $authuser, $worker);
74e09a93 TL	131	}});
74e09a93 TL	132
963c06bb DM	133	__PACKAGE__->register_method({
	134	name => 'nodes',
	135	path => 'nodes',
	136	method => 'GET',
	137	description => "Corosync node list.",
fb7c665a EK	138	permissions => {
	139	check => ['perm', '/', [ 'Sys.Audit' ]],
	140	},
963c06bb DM	141	parameters => {
	142	additionalProperties => 0,
	143	properties => {},
	144	},
	145	returns => {
	146	type => 'array',
	147	items => {
	148	type => "object",
	149	properties => {
	150	node => { type => 'string' },
	151	},
	152	},
	153	links => [ { rel => 'child', href => "{node}" } ],
	154	},
	155	code => sub {
	156	my ($param) = @_;
	157
	158
	159	my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
b6973a89	160	my $nodelist = PVE::Corosync::nodelist($conf);
963c06bb DM	161
	162	return PVE::RESTHandler::hash_to_array($nodelist, 'node');
	163	}});
	164
1d26c202 TL	165	# lock method to ensure local and cluster wide atomicity
	166	# if we're a single node cluster just lock locally, we have no other cluster
	167	# node which we could contend with, else also acquire a cluster wide lock
	168	my $config_change_lock = sub {
	169	my ($code) = @_;
	170
b184a69d	171	PVE::Tools::lock_file($local_cluster_lock, 10, sub {
1d26c202 TL	172	PVE::Cluster::cfs_update(1);
	173	my $members = PVE::Cluster::get_members();
	174	if (scalar(keys %$members) > 1) {
34b23d46	175	my $res = PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
7e192a44 TL	176
7e192a44 TL	177	# cfs_lock_file only sets $@ but lock_file doesn't propagates $@ unless we die here
34b23d46	178	die $@ if defined($@);
7e192a44	179
34b23d46	180	return $res;
1d26c202 TL	181	} else {
	182	return $code->();
	183	}
	184	});
	185	};
	186
1d26c202 TL	187	__PACKAGE__->register_method ({
	188	name => 'addnode',
	189	path => 'nodes/{node}',
	190	method => 'POST',
	191	protected => 1,
79cf5a70	192	description => "Adds a node to the cluster configuration. This call is for internal use.",
1d26c202 TL	193	parameters => {
	194	additionalProperties => 0,
	195	properties => {
	196	node => get_standard_option('pve-node'),
10c6810e	197	nodeid => get_standard_option('corosync-nodeid'),
1d26c202 TL	198	votes => {
	199	type => 'integer',
	200	description => "Number of votes for this node",
	201	minimum => 0,
	202	optional => 1,
	203	},
	204	force => {
	205	type => 'boolean',
	206	description => "Do not throw error if node already exists.",
	207	optional => 1,
	208	},
1584e3a1 TL	209	link0 => get_standard_option('corosync-link'),
1584e3a1 TL	210	link1 => get_standard_option('corosync-link'),
1d26c202 TL	211	},
1d26c202 TL	212	},
331d957b TL	213	returns => {
	214	type => "object",
	215	properties => {
	216	corosync_authkey => {
	217	type => 'string',
	218	},
	219	corosync_conf => {
	220	type => 'string',
	221	}
	222	},
	223	},
1d26c202 TL	224	code => sub {
	225	my ($param) = @_;
	226
	227	PVE::Cluster::check_cfs_quorum();
	228
	229	my $code = sub {
	230	my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
	231	my $nodelist = PVE::Corosync::nodelist($conf);
	232	my $totem_cfg = PVE::Corosync::totem_config($conf);
	233
	234	my $name = $param->{node};
	235
	236	# ensure we do not reuse an address, that can crash the whole cluster!
	237	my $check_duplicate_addr = sub {
1584e3a1 TL	238	my $link = shift;
	239	return if !defined($link) \|\| !defined($link->{address});
	240	my $addr = $link->{address};
1d26c202 TL	241
	242	while (my ($k, $v) = each %$nodelist) {
	243	next if $k eq $name; # allows re-adding a node if force is set
1584e3a1 TL	244
	245	for my $linknumber (0..1) {
	246	my $id = "ring${linknumber}_addr";
	247	next if !defined($v->{$id});
	248
	249	die "corosync: address '$addr' already used on link $id by node '$k'\n"
	250	if $v->{$id} eq $addr;
dd92cac5	251	}
1d26c202 TL	252	}
	253	};
	254
1584e3a1 TL	255	my $link0 = PVE::Cluster::parse_corosync_link($param->{link0});
	256	my $link1 = PVE::Cluster::parse_corosync_link($param->{link1});
	257
	258	$check_duplicate_addr->($link0);
	259	$check_duplicate_addr->($link1);
1d26c202	260
1584e3a1 TL	261	# FIXME: handle all links (0-7), they're all independent now
1584e3a1 TL	262	$link0->{address} //= $name if exists($totem_cfg->{interface}->{0});
1d26c202	263
1584e3a1 TL	264	die "corosync: using 'link1' parameter needs a interface with linknumber '1' configured!\n"
1584e3a1 TL	265	if $link1 && !defined($totem_cfg->{interface}->{1});
1d26c202	266
1584e3a1 TL	267	die "corosync: totem interface with linknumber 1 configured but 'link1' parameter not defined!\n"
1584e3a1 TL	268	if defined($totem_cfg->{interface}->{1}) && !defined($link1);
1d26c202 TL	269
	270	if (defined(my $res = $nodelist->{$name})) {
	271	$param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
	272	$param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
	273
	274	if ($res->{quorum_votes} == $param->{votes} &&
	275	$res->{nodeid} == $param->{nodeid} && $param->{force}) {
	276	print "forcing overwrite of configured node '$name'\n";
	277	} else {
	278	die "can't add existing node '$name'\n";
	279	}
	280	} elsif (!$param->{nodeid}) {
	281	my $nodeid = 1;
	282
	283	while(1) {
	284	my $found = 0;
	285	foreach my $v (values %$nodelist) {
	286	if ($v->{nodeid} eq $nodeid) {
	287	$found = 1;
	288	$nodeid++;
	289	last;
	290	}
	291	}
	292	last if !$found;
	293	};
	294
	295	$param->{nodeid} = $nodeid;
	296	}
	297
	298	$param->{votes} = 1 if !defined($param->{votes});
	299
	300	PVE::Cluster::gen_local_dirs($name);
	301
	302	eval { PVE::Cluster::ssh_merge_keys(); };
	303	warn $@ if $@;
	304
	305	$nodelist->{$name} = {
1584e3a1	306	ring0_addr => $link0->{address},
1d26c202 TL	307	nodeid => $param->{nodeid},
	308	name => $name,
	309	};
1584e3a1	310	$nodelist->{$name}->{ring1_addr} = $link1->{address} if defined($link1);
1d26c202 TL	311	$nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
1d26c202 TL	312
855671ec TL	313	PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster");
855671ec TL	314
1d26c202 TL	315	PVE::Corosync::update_nodelist($conf, $nodelist);
	316	};
	317
	318	$config_change_lock->($code);
	319	die $@ if $@;
	320
331d957b TL	321	my $res = {
	322	corosync_authkey => PVE::Tools::file_get_contents($authfile),
	323	corosync_conf => PVE::Tools::file_get_contents($clusterconf),
	324	};
	325
	326	return $res;
1d26c202 TL	327	}});
	328
	329
	330	__PACKAGE__->register_method ({
	331	name => 'delnode',
	332	path => 'nodes/{node}',
	333	method => 'DELETE',
	334	protected => 1,
	335	description => "Removes a node from the cluster configuration.",
	336	parameters => {
	337	additionalProperties => 0,
	338	properties => {
	339	node => get_standard_option('pve-node'),
	340	},
	341	},
	342	returns => { type => 'null' },
	343	code => sub {
	344	my ($param) = @_;
	345
	346	my $local_node = PVE::INotify::nodename();
	347	die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node;
	348
	349	PVE::Cluster::check_cfs_quorum();
	350
	351	my $code = sub {
	352	my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
	353	my $nodelist = PVE::Corosync::nodelist($conf);
	354
	355	my $node;
	356	my $nodeid;
	357
	358	foreach my $tmp_node (keys %$nodelist) {
	359	my $d = $nodelist->{$tmp_node};
	360	my $ring0_addr = $d->{ring0_addr};
	361	my $ring1_addr = $d->{ring1_addr};
	362	if (($tmp_node eq $param->{node}) \|\|
	363	(defined($ring0_addr) && ($ring0_addr eq $param->{node})) \|\|
	364	(defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
	365	$node = $tmp_node;
	366	$nodeid = $d->{nodeid};
	367	last;
	368	}
	369	}
	370
	371	die "Node/IP: $param->{node} is not a known host of the cluster.\n"
	372	if !defined($node);
	373
855671ec TL	374	PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster");
855671ec TL	375
1d26c202 TL	376	delete $nodelist->{$node};
	377
	378	PVE::Corosync::update_nodelist($conf, $nodelist);
	379
	380	PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid]) if defined($nodeid);
	381	};
	382
	383	$config_change_lock->($code);
	384	die $@ if $@;
	385
	386	return undef;
	387	}});
	388
fca7797d TL	389	__PACKAGE__->register_method ({
	390	name => 'join_info',
	391	path => 'join',
be85d00b TL	392	permissions => {
	393	check => ['perm', '/', [ 'Sys.Audit' ]],
	394	},
fca7797d TL	395	method => 'GET',
	396	description => "Get information needed to join this cluster over the connected node.",
	397	parameters => {
	398	additionalProperties => 0,
	399	properties => {
	400	node => get_standard_option('pve-node', {
	401	description => "The node for which the joinee gets the nodeinfo. ",
	402	default => "current connected node",
	403	optional => 1,
	404	}),
	405	},
	406	},
	407	returns => {
	408	type => 'object',
	409	additionalProperties => 0,
	410	properties => {
	411	nodelist => {
	412	type => 'array',
	413	items => {
	414	type => "object",
	415	additionalProperties => 1,
	416	properties => {
	417	name => get_standard_option('pve-node'),
	418	nodeid => get_standard_option('corosync-nodeid'),
3f1f4d68	419	ring0_addr => get_standard_option('corosync-link'),
fca7797d TL	420	quorum_votes => { type => 'integer', minimum => 0 },
	421	pve_addr => { type => 'string', format => 'ip' },
	422	pve_fp => get_standard_option('fingerprint-sha256'),
	423	},
	424	},
	425	},
	426	preferred_node => get_standard_option('pve-node'),
	427	totem => { type => 'object' },
	428	config_digest => { type => 'string' },
	429	},
	430	},
	431	code => sub {
	432	my ($param) = @_;
	433
	434	my $nodename = $param->{node} // PVE::INotify::nodename();
	435
	436	PVE::Cluster::cfs_update(1);
	437	my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
	438
	439	die "node is not in a cluster, no join info available!\n"
	440	if !($conf && $conf->{main});
	441
	442	my $totem_cfg = $conf->{main}->{totem} // {};
	443	my $nodelist = $conf->{main}->{nodelist}->{node} // {};
	444	my $corosync_config_digest = $conf->{digest};
	445
	446	die "unknown node '$nodename'\n" if ! $nodelist->{$nodename};
	447
	448	foreach my $name (keys %$nodelist) {
	449	my $node = $nodelist->{$name};
	450	$node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name);
	451	$node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name));
	452	}
	453
	454	my $res = {
	455	nodelist => [ values %$nodelist ],
	456	preferred_node => $nodename,
	457	totem => $totem_cfg,
	458	config_digest => $corosync_config_digest,
	459	};
	460
	461	return $res;
	462	}});
	463
6ed49eb1 TL	464	__PACKAGE__->register_method ({
	465	name => 'join',
	466	path => 'join',
	467	method => 'POST',
	468	protected => 1,
	469	description => "Joins this node into an existing cluster.",
	470	parameters => {
	471	additionalProperties => 0,
	472	properties => {
	473	hostname => {
	474	type => 'string',
	475	description => "Hostname (or IP) of an existing cluster member."
	476	},
10c6810e	477	nodeid => get_standard_option('corosync-nodeid'),
6ed49eb1 TL	478	votes => {
	479	type => 'integer',
	480	description => "Number of votes for this node",
	481	minimum => 0,
	482	optional => 1,
	483	},
	484	force => {
	485	type => 'boolean',
	486	description => "Do not throw error if node already exists.",
	487	optional => 1,
	488	},
1584e3a1	489	link0 => get_standard_option('corosync-link', {
072ba2e0	490	default => "IP resolved by node's hostname",
10c6810e	491	}),
1584e3a1	492	link1 => get_standard_option('corosync-link'),
6ed49eb1 TL	493	fingerprint => get_standard_option('fingerprint-sha256'),
	494	password => {
	495	description => "Superuser (root) password of peer node.",
	496	type => 'string',
	497	maxLength => 128,
	498	},
	499	},
	500	},
	501	returns => { type => 'string' },
	502	code => sub {
	503	my ($param) = @_;
	504
	505	my $rpcenv = PVE::RPCEnvironment::get();
	506	my $authuser = $rpcenv->get_user();
	507
	508	my $worker = sub {
17a4445f	509	STDOUT->autoflush();
b184a69d TL	510	PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::join, $param);
b184a69d TL	511	die $@ if $@;
6ed49eb1 TL	512	};
6ed49eb1 TL	513
8c4e30d3	514	return $rpcenv->fork_worker('clusterjoin', undef, $authuser, $worker);
6ed49eb1 TL	515	}});
	516
	517
963c06bb DM	518	__PACKAGE__->register_method({
	519	name => 'totem',
	520	path => 'totem',
	521	method => 'GET',
	522	description => "Get corosync totem protocol settings.",
fb7c665a EK	523	permissions => {
	524	check => ['perm', '/', [ 'Sys.Audit' ]],
	525	},
963c06bb DM	526	parameters => {
	527	additionalProperties => 0,
	528	properties => {},
	529	},
	530	returns => {
	531	type => "object",
	532	properties => {},
	533	},
	534	code => sub {
	535	my ($param) = @_;
	536
	537
	538	my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
	539
496de919 TL	540	my $totem_cfg = $conf->{main}->{totem};
	541
	542	return $totem_cfg;
963c06bb DM	543	}});
963c06bb DM	544
f0f8ee41 OB	545	__PACKAGE__->register_method ({
	546	name => 'status',
	547	path => 'qdevice',
	548	method => 'GET',
	549	description => 'Get QDevice status',
	550	permissions => {
	551	check => ['perm', '/', [ 'Sys.Audit' ]],
	552	},
	553	parameters => {
	554	additionalProperties => 0,
	555	properties => {},
	556	},
	557	returns => {
	558	type => "object",
	559	},
	560	code => sub {
	561	my ($param) = @_;
	562
	563	my $result = {};
	564	my $socket_path = "/var/run/corosync-qdevice/corosync-qdevice.sock";
	565	return $result if !-S $socket_path;
	566
	567	my $qdevice_socket = IO::Socket::UNIX->new(
	568	Type => SOCK_STREAM,
	569	Peer => $socket_path,
	570	);
	571
	572	print $qdevice_socket "status verbose\n";
	573	my $qdevice_keys = {
	574	"Algorithm" => 1,
	575	"Echo reply" => 1,
	576	"Last poll call" => 1,
	577	"Model" => 1,
	578	"QNetd host" => 1,
	579	"State" => 1,
	580	"Tie-breaker" => 1,
	581	};
	582	while (my $line = <$qdevice_socket>) {
	583	chomp $line;
	584	next if $line =~ /^\s/;
	585	if ($line =~ /^(.?)\s:\s(.)$/) {
	586	$result->{$1} = $2 if $qdevice_keys->{$1};
	587	}
	588	}
	589
	590	return $result;
	591	}});
	592	#TODO: possibly add setup and remove methods
	593
	594
963c06bb	595	1;