data/PVE/CLI/pvecm.pm

   1 package PVE::CLI::pvecm;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd qw(getcwd);
   7 use File::Path;
   8 use File::Basename;
   9 use PVE::Tools qw(run_command);
  10 use PVE::Cluster;
  11 use PVE::INotify;
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::RPCEnvironment;
  14 use PVE::CLIHandler;
  15 use PVE::PTY;
  16 use PVE::API2::ClusterConfig;
  17 use PVE::Corosync;
  18 use PVE::Cluster::Setup;
  19
  20 use base qw(PVE::CLIHandler);
  21
  22 $ENV{HOME} = '/root'; # for ssh-copy-id
  23
  24 my $basedir = "/etc/pve";
  25 my $clusterconf = "$basedir/corosync.conf";
  26 my $libdir = "/var/lib/pve-cluster";
  27 my $authfile = "/etc/corosync/authkey";
  28
  29
  30 sub setup_environment {
  31     PVE::RPCEnvironment->setup_default_cli_env();
  32 }
  33
  34 __PACKAGE__->register_method ({
  35     name => 'keygen',
  36     path => 'keygen',
  37     method => 'PUT',
  38     description => "Generate new cryptographic key for corosync.",
  39     parameters => {
  40         additionalProperties => 0,
  41         properties => {
  42             filename => {
  43                 type => 'string',
  44                 description => "Output file name"
  45             }
  46         },
  47     },
  48     returns => { type => 'null' },
  49
  50     code => sub {
  51         my ($param) = @_;
  52
  53         my $filename = $param->{filename};
  54
  55         # test EUID
  56         $> == 0 || die "Error: Authorization key must be generated as root user.\n";
  57         my $dirname = dirname($filename);
  58
  59         die "key file '$filename' already exists\n" if -e $filename;
  60
  61         File::Path::make_path($dirname) if $dirname;
  62
  63         run_command(['corosync-keygen', '-l', '-k', $filename]);
  64
  65         return undef;
  66     }});
  67
  68 my $foreach_member = sub {
  69     my ($code, $noerr) = @_;
  70
  71     my $members = PVE::Cluster::get_members();
  72     foreach my $node (sort keys %$members) {
  73         if (my $ip = $members->{$node}->{ip}) {
  74             $code->($node, $ip);
  75         } else {
  76             die "cannot get the cluster IP for node '$node'.\n" if !$noerr;
  77             warn "cannot get the cluster IP for node '$node'.\n";
  78             return undef;
  79         }
  80     }
  81 };
  82
  83 __PACKAGE__->register_method ({
  84     name => 'setup_qdevice',
  85     path => 'setup_qdevice',
  86     method => 'PUT',
  87     description => "Setup the use of a QDevice",
  88     parameters => {
  89         additionalProperties => 0,
  90         properties => {
  91             address => {
  92                 type => 'string', format => 'ip',
  93                 description => "Specifies the network address of an external corosync QDevice" ,
  94             },
  95             network => {
  96                 type => 'string',
  97                 format => 'CIDR',
  98                 description => 'The network which should be used to connect to the external qdevice',
  99                 optional => 1,
 100             },
 101             force => {
 102                 type => 'boolean',
 103                 description => "Do not throw error on possible dangerous operations.",
 104                 optional => 1,
 105             },
 106         },
 107     },
 108     returns => { type => 'null' },
 109
 110     code => sub {
 111         my ($param) = @_;
 112
 113         PVE::Corosync::check_conf_exists();
 114
 115         my $members = PVE::Cluster::get_members();
 116         foreach my $node (sort keys %$members) {
 117             die "All nodes must be online! Node $node is offline, aborting.\n"
 118                 if !$members->{$node}->{online};
 119         }
 120
 121         my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 122
 123         die "QDevice already configured!\n"
 124             if defined($conf->{main}->{quorum}->{device}) && !$param->{force};
 125
 126         my $network = $param->{network};
 127
 128         my $model = "net";
 129         my $algorithm = 'ffsplit';
 130         if (scalar(%{$members}) & 1) {
 131             if ($param->{force}) {
 132                 $algorithm = 'lms';
 133             } else {
 134                 die "Clusters with an odd node count are not officially supported!\n";
 135             }
 136         }
 137
 138         my $qnetd_addr = $param->{address};
 139         my $base_dir = "/etc/corosync/qdevice/net";
 140         my $db_dir_qnetd = "/etc/corosync/qnetd/nssdb";
 141         my $db_dir_node = "$base_dir/nssdb";
 142         my $ca_export_base = "qnetd-cacert.crt";
 143         my $ca_export_file = "$db_dir_qnetd/$ca_export_base";
 144         my $crq_file_base = "qdevice-net-node.crq";
 145         my $p12_file_base = "qdevice-net-node.p12";
 146         my $qdevice_certutil = "corosync-qdevice-net-certutil";
 147         my $qnetd_certutil= "corosync-qnetd-certutil";
 148         my $clustername = $conf->{main}->{totem}->{cluster_name};
 149
 150         run_command(['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$qnetd_addr"]);
 151
 152         if (-d $db_dir_node) {
 153             # FIXME: check on all nodes?!
 154             if ($param->{force}) {
 155                 rmtree $db_dir_node;
 156             } else {
 157                 die "QDevice certificate store already initialised, set force to delete!\n";
 158             }
 159         }
 160
 161         my $ssh_cmd = ['ssh', '-o', 'BatchMode=yes', '-lroot'];
 162         my $scp_cmd = ['scp', '-o', 'BatchMode=yes'];
 163
 164         print "\nINFO: initializing qnetd server\n";
 165         run_command(
 166             [@$ssh_cmd, $qnetd_addr, $qnetd_certutil, "-i"],
 167             noerr => 1
 168         );
 169
 170         print "\nINFO: copying CA cert and initializing on all nodes\n";
 171         run_command([@$scp_cmd, "root\@\[$qnetd_addr\]:$ca_export_file", "/etc/pve/$ca_export_base"]);
 172         $foreach_member->(sub {
 173             my ($node, $ip) = @_;
 174             my $outsub = sub { print "\nnode '$node': " . shift };
 175             run_command(
 176                 [@$ssh_cmd, $ip, $qdevice_certutil, "-i", "-c", "/etc/pve/$ca_export_base"],
 177                 noerr => 1, outfunc => \&$outsub
 178             );
 179         });
 180         unlink "/etc/pve/$ca_export_base";
 181
 182         print "\nINFO: generating cert request\n";
 183         run_command([$qdevice_certutil, "-r", "-n", $clustername]);
 184
 185         print "\nINFO: copying exported cert request to qnetd server\n";
 186         run_command([@$scp_cmd, "$db_dir_node/$crq_file_base", "root\@\[$qnetd_addr\]:/tmp"]);
 187
 188         print "\nINFO: sign and export cluster cert\n";
 189         run_command([
 190                 @$ssh_cmd, $qnetd_addr, $qnetd_certutil, "-s", "-c",
 191                 "/tmp/$crq_file_base", "-n", "$clustername"
 192             ]);
 193
 194         print "\nINFO: copy exported CRT\n";
 195         run_command([
 196                 @$scp_cmd, "root\@\[$qnetd_addr\]:$db_dir_qnetd/cluster-$clustername.crt",
 197                 "$db_dir_node"
 198             ]);
 199
 200         print "\nINFO: import certificate\n";
 201         run_command(["$qdevice_certutil", "-M", "-c", "$db_dir_node/cluster-$clustername.crt"]);
 202
 203         print "\nINFO: copy and import pk12 cert to all nodes\n";
 204         run_command([@$scp_cmd, "$db_dir_node/$p12_file_base", "/etc/pve/"]);
 205         $foreach_member->(sub {
 206             my ($node, $ip) = @_;
 207             my $outsub = sub { print "\nnode '$node': " . shift };
 208             run_command([
 209                     @$ssh_cmd, $ip, "$qdevice_certutil", "-m", "-c",
 210                     "/etc/pve/$p12_file_base"], outfunc => \&$outsub
 211                 );
 212         });
 213         unlink "/etc/pve/$p12_file_base";
 214
 215
 216         my $code = sub {
 217             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 218             my $quorum_section = $conf->{main}->{quorum};
 219
 220             die "Qdevice already configured, must be removed before setting up new one!\n"
 221                 if defined($quorum_section->{device}); # must not be forced!
 222
 223             my $qdev_section = {
 224                 model => $model,
 225                 "$model" => {
 226                     tls => 'on',
 227                     host => $qnetd_addr,
 228                     algorithm => $algorithm,
 229                 }
 230             };
 231             $qdev_section->{votes} = 1 if $algorithm eq 'ffsplit';
 232
 233             $quorum_section->{device} = $qdev_section;
 234
 235             PVE::Corosync::atomic_write_conf($conf);
 236         };
 237
 238         print "\nINFO: add QDevice to cluster configuration\n";
 239         PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
 240         die $@ if $@;
 241
 242         $foreach_member->(sub {
 243             my ($node, $ip) = @_;
 244             my $outsub = sub { print "\nnode '$node': " . shift };
 245             print "\nINFO: start and enable corosync qdevice daemon on node '$node'...\n";
 246             run_command([@$ssh_cmd, $ip, 'systemctl', 'start', 'corosync-qdevice'], outfunc => \&$outsub);
 247             run_command([@$ssh_cmd, $ip, 'systemctl', 'enable', 'corosync-qdevice'], outfunc => \&$outsub);
 248         });
 249
 250         run_command(['corosync-cfgtool', '-R']); # do cluster wide config reload
 251
 252         return undef;
 253 }});
 254
 255 __PACKAGE__->register_method ({
 256     name => 'remove_qdevice',
 257     path => 'remove_qdevice',
 258     method => 'DELETE',
 259     description => "Remove a configured QDevice",
 260     parameters => {
 261         additionalProperties => 0,
 262         properties => {},
 263     },
 264     returns => { type => 'null' },
 265
 266     code => sub {
 267         my ($param) = @_;
 268
 269         PVE::Corosync::check_conf_exists();
 270
 271         my $members = PVE::Cluster::get_members();
 272         foreach my $node (sort keys %$members) {
 273             die "All nodes must be online! Node $node is offline, aborting.\n"
 274                 if !$members->{$node}->{online};
 275         }
 276
 277         my $ssh_cmd = ['ssh', '-o', 'BatchMode=yes', '-lroot'];
 278
 279         my $code = sub {
 280             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 281             my $quorum_section = $conf->{main}->{quorum};
 282
 283             die "No QDevice configured!\n" if !defined($quorum_section->{device});
 284
 285             delete $quorum_section->{device};
 286
 287             PVE::Corosync::atomic_write_conf($conf);
 288
 289             # cleanup qdev state (cert storage)
 290             my $qdev_state_dir =  "/etc/corosync/qdevice";
 291             $foreach_member->(sub {
 292                 my (undef, $ip) = @_;
 293                 run_command([@$ssh_cmd, $ip, '--', 'rm', '-rf', $qdev_state_dir]);
 294             });
 295         };
 296
 297         PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
 298         die $@ if $@;
 299
 300         $foreach_member->(sub {
 301             my (undef, $ip) = @_;
 302             run_command([@$ssh_cmd, $ip, 'systemctl', 'stop', 'corosync-qdevice']);
 303             run_command([@$ssh_cmd, $ip, 'systemctl', 'disable', 'corosync-qdevice']);
 304         });
 305
 306         run_command(['corosync-cfgtool', '-R']);
 307
 308         print "\nRemoved Qdevice.\n";
 309
 310         return undef;
 311 }});
 312
 313 __PACKAGE__->register_method ({
 314     name => 'add',
 315     path => 'add',
 316     method => 'PUT',
 317     description => "Adds the current node to an existing cluster.",
 318     parameters => {
 319         additionalProperties => 0,
 320         properties => PVE::Corosync::add_corosync_link_properties({
 321             hostname => {
 322                 type => 'string',
 323                 description => "Hostname (or IP) of an existing cluster member."
 324             },
 325             nodeid => get_standard_option('corosync-nodeid'),
 326             votes => {
 327                 type => 'integer',
 328                 description => "Number of votes for this node",
 329                 minimum => 0,
 330                 optional => 1,
 331             },
 332             force => {
 333                 type => 'boolean',
 334                 description => "Do not throw error if node already exists.",
 335                 optional => 1,
 336             },
 337             fingerprint => get_standard_option('fingerprint-sha256', {
 338                 optional => 1,
 339             }),
 340             'use_ssh' => {
 341                 type => 'boolean',
 342                 description => "Always use SSH to join, even if peer may do it over API.",
 343                 optional => 1,
 344             },
 345         }),
 346     },
 347     returns => { type => 'null' },
 348
 349     code => sub {
 350         my ($param) = @_;
 351
 352         # avoid "transport endpoint not connected" errors that occur if
 353         # restarting pmxcfs while in fuse-mounted /etc/pve
 354         die "Navigate out of $basedir before running 'pvecm add', for example by running 'cd'.\n"
 355             if getcwd() =~ m!^$basedir(/.*)?$!;
 356
 357         my $nodename = PVE::INotify::nodename();
 358         my $host = $param->{hostname};
 359
 360         my $worker = sub {
 361
 362             if (!$param->{use_ssh}) {
 363                 my $password = PVE::PTY::read_password("Please enter superuser (root) password for '$host': ");
 364
 365                 delete $param->{use_ssh};
 366                 $param->{password} = $password;
 367
 368                 my $local_cluster_lock = "/var/lock/pvecm.lock";
 369                 PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::Setup::join, $param);
 370
 371                 if (my $err = $@) {
 372                     if (ref($err) eq 'PVE::APIClient::Exception' && defined($err->{code}) && $err->{code} == 501) {
 373                         $err = "Remote side is not able to use API for Cluster join!\n" .
 374                                "Pass the 'use_ssh' switch or update the remote side.\n";
 375                     }
 376                     die $err;
 377                 }
 378                 return; # all OK, the API join endpoint successfully set us up
 379             }
 380
 381             # allow fallback to old ssh only join if wished or needed
 382
 383             my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 384             my $links = PVE::Corosync::extract_corosync_link_args($param);
 385
 386             PVE::Cluster::Setup::assert_joinable($local_ip_address, $links, $param->{force});
 387
 388             PVE::Cluster::Setup::setup_sshd_config();
 389             PVE::Cluster::Setup::setup_rootsshconfig();
 390             PVE::Cluster::Setup::setup_ssh_keys();
 391
 392             # make sure known_hosts is on local filesystem
 393             PVE::Cluster::Setup::ssh_unmerge_known_hosts();
 394
 395             my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"];
 396             run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
 397                                     'errmsg' => "unable to copy ssh ID");
 398
 399             $cmd = ['ssh', $host, '-o', 'BatchMode=yes', 'pvecm', 'apiver'];
 400             my $remote_apiver = 0;
 401             run_command($cmd, 'outfunc' => sub {
 402                 $remote_apiver = shift;
 403                 chomp $remote_apiver;
 404             }, 'noerr' => 1);
 405
 406             PVE::Cluster::Setup::assert_we_can_join_cluster_version($remote_apiver);
 407
 408             $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
 409                     'pvecm', 'addnode', $nodename, '--force', 1];
 410
 411             push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
 412             push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
 413
 414             my $link_desc = get_standard_option('corosync-link');
 415
 416             foreach my $link (keys %$links) {
 417                 push @$cmd, "--link$link", PVE::JSONSchema::print_property_string(
 418                     $links->{$link}, $link_desc->{format});
 419             }
 420
 421             # this will be used as fallback if no links are specified
 422             if (!%$links) {
 423                 push @$cmd, '--link0', $local_ip_address if $remote_apiver == 0;
 424                 push @$cmd, '--new_node_ip', $local_ip_address if $remote_apiver >= 1;
 425
 426                 print "No cluster network links passed explicitly, fallback to local node"
 427                     . " IP '$local_ip_address'\n";
 428             }
 429
 430             if (system (@$cmd) != 0) {
 431                 my $cmdtxt = join (' ', @$cmd);
 432                 die "unable to add node: command failed ($cmdtxt)\n";
 433             }
 434
 435             my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
 436             mkdir $tmpdir;
 437
 438             eval {
 439                 print "copy corosync auth key\n";
 440                 $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
 441                         "[$host]:$authfile $clusterconf", $tmpdir];
 442
 443                 system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
 444
 445                 my $corosync_conf = PVE::Tools::file_get_contents("$tmpdir/corosync.conf");
 446                 my $corosync_authkey = PVE::Tools::file_get_contents("$tmpdir/authkey");
 447
 448                 PVE::Cluster::Setup::finish_join($nodename, $corosync_conf, $corosync_authkey);
 449             };
 450             my $err = $@;
 451
 452             rmtree $tmpdir;
 453
 454             die $err if $err;
 455         };
 456
 457         # use a synced worker so we get a nice task log when joining through CLI
 458         my $rpcenv = PVE::RPCEnvironment::get();
 459         my $authuser = $rpcenv->get_user();
 460
 461         $rpcenv->fork_worker('clusterjoin', '',  $authuser, $worker);
 462
 463         return undef;
 464     }});
 465
 466 __PACKAGE__->register_method ({
 467     name => 'status',
 468     path => 'status',
 469     method => 'GET',
 470     description => "Displays the local view of the cluster status.",
 471     parameters => {
 472         additionalProperties => 0,
 473         properties => {},
 474     },
 475     returns => { type => 'null' },
 476
 477     code => sub {
 478         my ($param) = @_;
 479
 480         PVE::Corosync::check_conf_exists();
 481         my $conf = eval { PVE::Cluster::cfs_read_file("corosync.conf") } // {};
 482         warn "$@" if $@;
 483         my $totem = PVE::Corosync::totem_config($conf);
 484
 485         if (scalar(%$totem)) {
 486             my $print_info = sub {
 487                 my ($label, $key, $default) = @_;
 488                 my $val = $totem->{$key} // $default;
 489                 printf "%-17s %s\n", "$label:", "$val";
 490             };
 491
 492             printf "Cluster information\n";
 493             printf "-------------------\n";
 494             $print_info->('Name', 'cluster_name', 'UNKOWN?');
 495             $print_info->('Config Version', 'config_version', -1);
 496             $print_info->('Transport', 'transport', 'knet');
 497             $print_info->('Secure auth', 'secauth', 'off');
 498             printf "\n";
 499         }
 500
 501         exec ('corosync-quorumtool', '-siH');
 502         exit (-1); # should not be reached
 503     }});
 504
 505 __PACKAGE__->register_method ({
 506     name => 'nodes',
 507     path => 'nodes',
 508     method => 'GET',
 509     description => "Displays the local view of the cluster nodes.",
 510     parameters => {
 511         additionalProperties => 0,
 512         properties => {},
 513     },
 514     returns => { type => 'null' },
 515
 516     code => sub {
 517         my ($param) = @_;
 518
 519         PVE::Corosync::check_conf_exists();
 520
 521         exec ('corosync-quorumtool', '-l');
 522         exit (-1); # should not be reached
 523     }});
 524
 525 __PACKAGE__->register_method ({
 526     name => 'expected',
 527     path => 'expected',
 528     method => 'PUT',
 529     description => "Tells corosync a new value of expected votes.",
 530     parameters => {
 531         additionalProperties => 0,
 532         properties => {
 533             expected => {
 534                 type => 'integer',
 535                 description => "Expected votes",
 536                 minimum => 1,
 537             },
 538         },
 539     },
 540     returns => { type => 'null' },
 541
 542     code => sub {
 543         my ($param) = @_;
 544
 545         PVE::Corosync::check_conf_exists();
 546
 547         exec ('corosync-quorumtool', '-e', $param->{expected});
 548         exit (-1); # should not be reached
 549     }});
 550
 551 __PACKAGE__->register_method ({
 552     name => 'updatecerts',
 553     path => 'updatecerts',
 554     method => 'PUT',
 555     description => "Update node certificates (and generate all needed files/directories).",
 556     parameters => {
 557         additionalProperties => 0,
 558         properties => {
 559             force => {
 560                 description => "Force generation of new SSL certificate.",
 561                 type => 'boolean',
 562                 optional => 1,
 563             },
 564             silent => {
 565                 description => "Ignore errors (i.e. when cluster has no quorum).",
 566                 type => 'boolean',
 567                 optional => 1,
 568             },
 569         },
 570     },
 571     returns => { type => 'null' },
 572     code => sub {
 573         my ($param) = @_;
 574
 575         # we get called by the pve-cluster.service ExecStartPost and as we do
 576         # IO (on /etc/pve) which can hang (uninterruptedly D state). That'd be
 577         # no-good for ExecStartPost as it fails the whole service in this case
 578         PVE::Tools::run_fork_with_timeout(30, sub {
 579             PVE::Cluster::Setup::updatecerts_and_ssh($param->@{qw(force silent)});
 580             PVE::Cluster::prepare_observed_file_basedirs();
 581         });
 582
 583         return undef;
 584     }});
 585
 586 __PACKAGE__->register_method ({
 587     name => 'mtunnel',
 588     path => 'mtunnel',
 589     method => 'POST',
 590     description => "Used by VM/CT migration - do not use manually.",
 591     parameters => {
 592         additionalProperties => 0,
 593         properties => {
 594             get_migration_ip => {
 595                 type => 'boolean',
 596                 default => 0,
 597                 description => 'return the migration IP, if configured',
 598                 optional => 1,
 599             },
 600             migration_network => {
 601                 type => 'string',
 602                 format => 'CIDR',
 603                 description => 'the migration network used to detect the local migration IP',
 604                 optional => 1,
 605             },
 606             'run-command' => {
 607                 type => 'boolean',
 608                 description => 'Run a command with a tcp socket as standard input.'
 609                               .' The IP address and port are printed via this'
 610                               ." command's stdandard output first, each on a separate line.",
 611                 optional => 1,
 612             },
 613             'extra-args' => PVE::JSONSchema::get_standard_option('extra-args'),
 614         },
 615     },
 616     returns => { type => 'null'},
 617     code => sub {
 618         my ($param) = @_;
 619
 620         if (!PVE::Cluster::check_cfs_quorum(1)) {
 621             print "no quorum\n";
 622             return undef;
 623         }
 624
 625         my $get_local_migration_ip = sub {
 626             my ($cidr) = @_;
 627
 628             if (!defined($cidr)) {
 629                 my $dc_conf = cfs_read_file('datacenter.cfg');
 630                 $cidr = $dc_conf->{migration}->{network}
 631                 if defined($dc_conf->{migration}->{network});
 632             }
 633
 634             if (defined($cidr)) {
 635                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
 636
 637                 die "could not get migration ip: no IP address configured on local " .
 638                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
 639
 640                 die "could not get migration ip: multiple, different, IP address configured for " .
 641                     "network '$cidr'\n" if scalar(@$ips) > 1 && grep { @$ips[0] ne $_ } @$ips;
 642
 643                 return @$ips[0];
 644             }
 645
 646             return undef;
 647         };
 648
 649         my $network = $param->{migration_network};
 650         if ($param->{get_migration_ip}) {
 651             die "cannot use --run-command with --get_migration_ip\n"
 652                 if $param->{'run-command'};
 653
 654             if (my $ip = $get_local_migration_ip->($network)) {
 655                 print "ip: '$ip'\n";
 656             } else {
 657                 print "no ip\n";
 658             }
 659             # do not keep tunnel open when asked for migration ip
 660             return undef;
 661         }
 662
 663         if ($param->{'run-command'}) {
 664             my $cmd = $param->{'extra-args'};
 665             die "missing command\n"
 666                 if !$cmd || !scalar(@$cmd);
 667
 668             # Get an ip address to listen on, and find a free migration port
 669             my ($ip, $family);
 670             if (defined($network)) {
 671                 $ip = $get_local_migration_ip->($network)
 672                     or die "failed to get migration IP address to listen on\n";
 673                 $family = PVE::Tools::get_host_address_family($ip);
 674             } else {
 675                 my $nodename = PVE::INotify::nodename();
 676                 ($ip, $family) = PVE::Network::get_ip_from_hostname($nodename, 0);
 677             }
 678             my $port = PVE::Tools::next_migrate_port($family, $ip);
 679
 680             PVE::Tools::pipe_socket_to_command($cmd, $ip, $port);
 681             return undef;
 682         }
 683
 684         print "tunnel online\n";
 685         *STDOUT->flush();
 686
 687         while (my $line = <STDIN>) {
 688             chomp $line;
 689             last if $line =~ m/^quit$/;
 690         }
 691
 692         return undef;
 693     }});
 694
 695
 696 our $cmddef = {
 697     apiver => [ 'PVE::API2::ClusterConfig', 'join_api_version', [], {}, sub {
 698         my $apiver = shift;
 699         print "$apiver\n";
 700     }],
 701     keygen => [ __PACKAGE__, 'keygen', ['filename']],
 702     create => [ 'PVE::API2::ClusterConfig', 'create', ['clustername']],
 703     add => [ __PACKAGE__, 'add', ['hostname']],
 704     addnode => [ 'PVE::API2::ClusterConfig', 'addnode', ['node']],
 705     delnode => [ 'PVE::API2::ClusterConfig', 'delnode', ['node']],
 706     status => [ __PACKAGE__, 'status' ],
 707     nodes => [ __PACKAGE__, 'nodes' ],
 708     expected => [ __PACKAGE__, 'expected', ['expected']],
 709     updatecerts => [ __PACKAGE__, 'updatecerts', []],
 710     mtunnel => [ __PACKAGE__, 'mtunnel', ['extra-args']],
 711     qdevice => {
 712         setup => [ __PACKAGE__, 'setup_qdevice', ['address']],
 713         remove => [ __PACKAGE__, 'remove_qdevice', []],
 714     }
 715 };
 716
 717 1;