From a755ff5422f9218d8e5bf8441d94d127f70c6e30 Mon Sep 17 00:00:00 2001 From: Stefan Reiter Date: Thu, 9 Jan 2020 16:31:36 +0100 Subject: [PATCH 1/1] Add cluster join API version check Adds API call GET /cluster/config/apiversion to retrieve remote clusters join-API version (0 is assumed for versions without this endpoint). Also available via CLI as 'pvecm apiver'. Introduce API_AGE similar to storage plugin API, but with two ages for cluster/joinee roles. Currently, all versions are intercompatible. For future usage, a new 'addnode' parameter 'apiversion' is introduced, to allow introducing API breakages for joining nodes as well. As a first compatibility check, use new fallback method only if available. This ensures full compatibility between nodes/clusters with and without new fallback behaviour. Signed-off-by: Stefan Reiter --- data/PVE/API2/ClusterConfig.pm | 37 +++++++++++++++++++++++++++++++++- data/PVE/CLI/pvecm.pm | 22 +++++++++++++++++++- data/PVE/Cluster/Setup.pm | 23 ++++++++++++++++++++- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/data/PVE/API2/ClusterConfig.pm b/data/PVE/API2/ClusterConfig.pm index e6b59cf..205252f 100644 --- a/data/PVE/API2/ClusterConfig.pm +++ b/data/PVE/API2/ClusterConfig.pm @@ -58,11 +58,33 @@ __PACKAGE__->register_method({ { name => 'totem' }, { name => 'join' }, { name => 'qdevice' }, + { name => 'apiversion' }, ]; return $result; }}); +__PACKAGE__->register_method ({ + name => 'join_api_version', + path => 'apiversion', + method => 'GET', + description => "Return the version of the cluster join API available on this node.", + permissions => { + check => ['perm', '/', [ 'Sys.Audit' ]], + }, + parameters => { + additionalProperties => 0, + properties => {}, + }, + returns => { + type => 'integer', + minimum => 0, + description => "Cluster Join API version, currently " . PVE::Cluster::Setup::JOIN_API_VERSION, + }, + code => sub { + return PVE::Cluster::Setup::JOIN_API_VERSION; + }}); + __PACKAGE__->register_method ({ name => 'create', path => '', @@ -213,6 +235,11 @@ __PACKAGE__->register_method ({ format => 'ip', optional => 1, }, + apiversion => { + type => 'integer', + description => 'The JOIN_API_VERSION of the new node.', + optional => 1, + }, }), }, returns => { @@ -235,6 +262,14 @@ __PACKAGE__->register_method ({ code => sub { my ($param) = @_; + $param->{apiversion} //= 0; + if ($param->{apiversion} < (PVE::Cluster::Setup::JOIN_API_VERSION - + PVE::Cluster::Setup::JOIN_API_AGE_AS_CLUSTER)) { + die "unsupported old API version on joining node ($param->{apiversion}," + . " cluster node has " . PVE::Cluster::Setup::JOIN_API_VERSION + . "), please upgrade before joining\n"; + } + PVE::Cluster::check_cfs_quorum(); my $vc_errors; @@ -291,7 +326,7 @@ __PACKAGE__->register_method ({ # FIXME: remove in 8.0 or when joining an old node not supporting # new_node_ip becomes infeasible otherwise my $legacy_fallback = 0; - if (!$param->{new_node_ip} && scalar(%$links) == 1) { + if (!$param->{new_node_ip} && scalar(%$links) == 1 && $param->{apiversion} == 0) { my $passed_link_id = (keys %$links)[0]; my $passed_link = delete $links->{$passed_link_id}; $param->{new_node_ip} = $passed_link->{address}; diff --git a/data/PVE/CLI/pvecm.pm b/data/PVE/CLI/pvecm.pm index a558813..f205b8d 100755 --- a/data/PVE/CLI/pvecm.pm +++ b/data/PVE/CLI/pvecm.pm @@ -390,6 +390,20 @@ __PACKAGE__->register_method ({ run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {}, 'errmsg' => "unable to copy ssh ID"); + $cmd = ['ssh', $host, '-o', 'BatchMode=yes', 'pvecm', 'apiver']; + my $remote_apiver = 0; + run_command($cmd, 'outfunc' => sub { + $remote_apiver = shift; + chomp $remote_apiver; + }, 'noerr' => 1); + + if ($remote_apiver < (PVE::Cluster::Setup::JOIN_API_VERSION - + PVE::Cluster::Setup::JOIN_API_AGE_AS_JOINEE)) { + die "error: incompatible join API version on cluster ($remote_apiver," + . " local has " . PVE::Cluster::Setup::JOIN_API_VERSION . "). Make" + . " sure all nodes are up-to-date.\n"; + } + $cmd = ['ssh', $host, '-o', 'BatchMode=yes', 'pvecm', 'addnode', $nodename, '--force', 1]; @@ -403,7 +417,9 @@ __PACKAGE__->register_method ({ # this will be used as fallback if no links are specified if (!%$links) { - push @$cmd, '--link0', $local_ip_address; + push @$cmd, '--link0', $local_ip_address if $remote_apiver == 0; + push @$cmd, '--new_node_ip', $local_ip_address if $remote_apiver >= 1; + print "No cluster network links passed explicitly, fallback to local node" . " IP '$local_ip_address'\n"; } @@ -674,6 +690,10 @@ __PACKAGE__->register_method ({ our $cmddef = { + apiver => [ 'PVE::API2::ClusterConfig', 'join_api_version', [], {}, sub { + my $apiver = shift; + print "$apiver\n"; + }], keygen => [ __PACKAGE__, 'keygen', ['filename']], create => [ 'PVE::API2::ClusterConfig', 'create', ['clustername']], add => [ __PACKAGE__, 'add', ['hostname']], diff --git a/data/PVE/Cluster/Setup.pm b/data/PVE/Cluster/Setup.pm index 1bf5b1f..72289c0 100644 --- a/data/PVE/Cluster/Setup.pm +++ b/data/PVE/Cluster/Setup.pm @@ -20,6 +20,13 @@ use PVE::Network; use PVE::Tools; use PVE::Certificate; +# Only relevant for pre-join checks, after join happened versions can differ +use constant JOIN_API_VERSION => 1; +# (APIVER-this) is oldest version a new node in addnode can have to be accepted +use constant JOIN_API_AGE_AS_CLUSTER => 1; +# (APIVER-this) is oldest version a cluster node can have to still try joining +use constant JOIN_API_AGE_AS_JOINEE => 1; + my $pmxcfs_base_dir = PVE::Cluster::base_dir(); my $pmxcfs_auth_dir = PVE::Cluster::auth_dir(); @@ -677,6 +684,14 @@ sub join { # login raises an exception on failure, so if we get here we're good print "Login succeeded.\n"; + # check cluster join API version + my $apiver = eval { $conn->get("/cluster/config/apiversion") } // 0; + + if ($apiver < (JOIN_API_VERSION - JOIN_API_AGE_AS_JOINEE)) { + die "error: incompatible join API version on cluster ($apiver, local has " + . JOIN_API_VERSION . "). Make sure all nodes are up-to-date.\n"; + } + my $args = {}; $args->{force} = $param->{force} if defined($param->{force}); $args->{nodeid} = $param->{nodeid} if $param->{nodeid}; @@ -687,11 +702,17 @@ sub join { # this will be used as fallback if no links are specified if (!%$links) { - $args->{link0} = $local_ip_address; + $args->{link0} = $local_ip_address if $apiver == 0; + $args->{new_node_ip} = $local_ip_address if $apiver >= 1; + print "No cluster network links passed explicitly, fallback to local node" . " IP '$local_ip_address'\n"; } + if ($apiver >= 1) { + $args->{apiversion} = JOIN_API_VERSION; + } + print "Request addition of this node\n"; my $res = eval { $conn->post("/cluster/config/nodes/$nodename", $args); }; if (my $err = $@) { -- 2.39.2