Add cluster join API version check

Adds API call GET /cluster/config/apiversion to retrieve remote clusters
join-API version (0 is assumed for versions without this endpoint). Also
available via CLI as 'pvecm apiver'.

Introduce API_AGE similar to storage plugin API, but with two ages for
cluster/joinee roles. Currently, all versions are intercompatible.

For future usage, a new 'addnode' parameter 'apiversion' is introduced,
to allow introducing API breakages for joining nodes as well.

As a first compatibility check, use new fallback method only if
available. This ensures full compatibility between nodes/clusters with
and without new fallback behaviour.

Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>

diff --git a/data/PVE/API2/ClusterConfig.pm b/data/PVE/API2/ClusterConfig.pm
index e6b59cf6db91c7b67147f6f37e1c5765f10a03d1..205252fa06e52ffac39f6a901913aa1eed247202 100644 (file)
--- a/data/PVE/API2/ClusterConfig.pm
+++ b/data/PVE/API2/ClusterConfig.pm
@@ -58,11 +58,33 @@ __PACKAGE__->register_method({
            { name => 'totem' },
            { name => 'join' },
            { name => 'qdevice' },
+           { name => 'apiversion' },
        ];
 
        return $result;
     }});
 
+__PACKAGE__->register_method ({
+    name => 'join_api_version',
+    path => 'apiversion',
+    method => 'GET',
+    description => "Return the version of the cluster join API available on this node.",
+    permissions => {
+       check => ['perm', '/', [ 'Sys.Audit' ]],
+    },
+    parameters => {
+       additionalProperties => 0,
+       properties => {},
+    },
+    returns => {
+       type => 'integer',
+       minimum => 0,
+       description => "Cluster Join API version, currently " . PVE::Cluster::Setup::JOIN_API_VERSION,
+    },
+    code => sub {
+       return PVE::Cluster::Setup::JOIN_API_VERSION;
+    }});
+
 __PACKAGE__->register_method ({
     name => 'create',
     path => '',
@@ -213,6 +235,11 @@ __PACKAGE__->register_method ({
                format => 'ip',
                optional => 1,
            },
+           apiversion => {
+               type => 'integer',
+               description => 'The JOIN_API_VERSION of the new node.',
+               optional => 1,
+           },
        }),
     },
     returns => {
@@ -235,6 +262,14 @@ __PACKAGE__->register_method ({
     code => sub {
        my ($param) = @_;
 
+       $param->{apiversion} //= 0;
+       if ($param->{apiversion} < (PVE::Cluster::Setup::JOIN_API_VERSION -
+           PVE::Cluster::Setup::JOIN_API_AGE_AS_CLUSTER)) {
+           die "unsupported old API version on joining node ($param->{apiversion},"
+               . " cluster node has " . PVE::Cluster::Setup::JOIN_API_VERSION
+               . "), please upgrade before joining\n";
+       }
+
        PVE::Cluster::check_cfs_quorum();
 
        my $vc_errors;
@@ -291,7 +326,7 @@ __PACKAGE__->register_method ({
            # FIXME: remove in 8.0 or when joining an old node not supporting
            # new_node_ip becomes infeasible otherwise
            my $legacy_fallback = 0;
-           if (!$param->{new_node_ip} && scalar(%$links) == 1) {
+           if (!$param->{new_node_ip} && scalar(%$links) == 1 && $param->{apiversion} == 0) {
                my $passed_link_id = (keys %$links)[0];
                my $passed_link = delete $links->{$passed_link_id};
                $param->{new_node_ip} = $passed_link->{address};

diff --git a/data/PVE/CLI/pvecm.pm b/data/PVE/CLI/pvecm.pm
index a558813bd6e8cd89388fb329d204b5fa3f63692a..f205b8d38c04d3750252569176088d4bbbb4e89c 100755 (executable)
--- a/data/PVE/CLI/pvecm.pm
+++ b/data/PVE/CLI/pvecm.pm
@@ -390,6 +390,20 @@ __PACKAGE__->register_method ({
            run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
                                    'errmsg' => "unable to copy ssh ID");
 
+           $cmd = ['ssh', $host, '-o', 'BatchMode=yes', 'pvecm', 'apiver'];
+           my $remote_apiver = 0;
+           run_command($cmd, 'outfunc' => sub {
+               $remote_apiver = shift;
+               chomp $remote_apiver;
+           }, 'noerr' => 1);
+
+           if ($remote_apiver < (PVE::Cluster::Setup::JOIN_API_VERSION -
+               PVE::Cluster::Setup::JOIN_API_AGE_AS_JOINEE)) {
+               die "error: incompatible join API version on cluster ($remote_apiver,"
+                   . " local has " . PVE::Cluster::Setup::JOIN_API_VERSION . "). Make"
+                   . " sure all nodes are up-to-date.\n";
+           }
+
            $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
                    'pvecm', 'addnode', $nodename, '--force', 1];
 
@@ -403,7 +417,9 @@ __PACKAGE__->register_method ({
 
            # this will be used as fallback if no links are specified
            if (!%$links) {
-               push @$cmd, '--link0', $local_ip_address;
+               push @$cmd, '--link0', $local_ip_address if $remote_apiver == 0;
+               push @$cmd, '--new_node_ip', $local_ip_address if $remote_apiver >= 1;
+
                print "No cluster network links passed explicitly, fallback to local node"
                    . " IP '$local_ip_address'\n";
            }
@@ -674,6 +690,10 @@ __PACKAGE__->register_method ({
 
 
 our $cmddef = {
+    apiver => [ 'PVE::API2::ClusterConfig', 'join_api_version', [], {}, sub {
+       my $apiver = shift;
+       print "$apiver\n";
+    }],
     keygen => [ __PACKAGE__, 'keygen', ['filename']],
     create => [ 'PVE::API2::ClusterConfig', 'create', ['clustername']],
     add => [ __PACKAGE__, 'add', ['hostname']],

diff --git a/data/PVE/Cluster/Setup.pm b/data/PVE/Cluster/Setup.pm
index 1bf5b1f093ffa0276ef0c3c84d9ad1aa533a9978..72289c006424528a8c4456243c9ce03e382d55d5 100644 (file)
--- a/data/PVE/Cluster/Setup.pm
+++ b/data/PVE/Cluster/Setup.pm
@@ -20,6 +20,13 @@ use PVE::Network;
 use PVE::Tools;
 use PVE::Certificate;
 
+# Only relevant for pre-join checks, after join happened versions can differ
+use constant JOIN_API_VERSION => 1;
+# (APIVER-this) is oldest version a new node in addnode can have to be accepted
+use constant JOIN_API_AGE_AS_CLUSTER => 1;
+# (APIVER-this) is oldest version a cluster node can have to still try joining
+use constant JOIN_API_AGE_AS_JOINEE => 1;
+
 my $pmxcfs_base_dir = PVE::Cluster::base_dir();
 my $pmxcfs_auth_dir = PVE::Cluster::auth_dir();
 
@@ -677,6 +684,14 @@ sub join {
     # login raises an exception on failure, so if we get here we're good
     print "Login succeeded.\n";
 
+    # check cluster join API version
+    my $apiver = eval { $conn->get("/cluster/config/apiversion") } // 0;
+
+    if ($apiver < (JOIN_API_VERSION - JOIN_API_AGE_AS_JOINEE)) {
+       die "error: incompatible join API version on cluster ($apiver, local has "
+           . JOIN_API_VERSION . "). Make sure all nodes are up-to-date.\n";
+    }
+
     my $args = {};
     $args->{force} = $param->{force} if defined($param->{force});
     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
@@ -687,11 +702,17 @@ sub join {
 
     # this will be used as fallback if no links are specified
     if (!%$links) {
-       $args->{link0} = $local_ip_address;
+       $args->{link0} = $local_ip_address if $apiver == 0;
+       $args->{new_node_ip} = $local_ip_address if $apiver >= 1;
+
        print "No cluster network links passed explicitly, fallback to local node"
            . " IP '$local_ip_address'\n";
     }
 
+    if ($apiver >= 1) {
+       $args->{apiversion} = JOIN_API_VERSION;
+    }
+
     print "Request addition of this node\n";
     my $res = eval { $conn->post("/cluster/config/nodes/$nodename", $args); };
     if (my $err = $@) {