data/PVE/CLI/pvecm.pm

   1 package PVE::CLI::pvecm;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Net::IP;
   7 use File::Path;
   8 use File::Basename;
   9 use PVE::Tools qw(run_command);
  10 use PVE::Cluster;
  11 use PVE::INotify;
  12 use PVE::JSONSchema;
  13 use PVE::CLIHandler;
  14 use PVE::API2::ClusterConfig;
  15 use PVE::Corosync;
  16
  17 use base qw(PVE::CLIHandler);
  18
  19 $ENV{HOME} = '/root'; # for ssh-copy-id
  20
  21 my $basedir = "/etc/pve";
  22 my $clusterconf = "$basedir/corosync.conf";
  23 my $libdir = "/var/lib/pve-cluster";
  24 my $backupdir = "/var/lib/pve-cluster/backup";
  25 my $dbfile = "$libdir/config.db";
  26 my $authfile = "/etc/corosync/authkey";
  27
  28 sub backup_database {
  29
  30     print "backup old database\n";
  31
  32     mkdir $backupdir;
  33
  34     my $ctime = time();
  35     my $cmd = [
  36         ['echo', '.dump'],
  37         ['sqlite3', $dbfile],
  38         ['gzip', '-', \ ">${backupdir}/config-${ctime}.sql.gz"],
  39     ];
  40
  41     run_command($cmd, 'errmsg' => "cannot backup old database\n");
  42
  43     # purge older backup
  44     my $maxfiles = 10;
  45
  46     my @bklist = ();
  47     foreach my $fn (<$backupdir/config-*.sql.gz>) {
  48         if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
  49             push @bklist, [$fn, $1];
  50         }
  51     }
  52
  53     @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
  54
  55     while (scalar (@bklist) >= $maxfiles) {
  56         my $d = pop @bklist;
  57         print "delete old backup '$d->[0]'\n";
  58         unlink $d->[0];
  59     }
  60 }
  61
  62
  63 __PACKAGE__->register_method ({
  64     name => 'keygen',
  65     path => 'keygen',
  66     method => 'PUT',
  67     description => "Generate new cryptographic key for corosync.",
  68     parameters => {
  69         additionalProperties => 0,
  70         properties => {
  71             filename => {
  72                 type => 'string',
  73                 description => "Output file name"
  74             }
  75         },
  76     },
  77     returns => { type => 'null' },
  78
  79     code => sub {
  80         my ($param) = @_;
  81
  82         my $filename = $param->{filename};
  83
  84         # test EUID
  85         $> == 0 || die "Error: Authorization key must be generated as root user.\n";
  86         my $dirname = dirname($filename);
  87
  88         die "key file '$filename' already exists\n" if -e $filename;
  89
  90         File::Path::make_path($dirname) if $dirname;
  91
  92         run_command(['corosync-keygen', '-l', '-k', $filename]);
  93
  94         return undef;
  95     }});
  96
  97 __PACKAGE__->register_method ({
  98     name => 'create',
  99     path => 'create',
 100     method => 'PUT',
 101     description => "Generate new cluster configuration.",
 102     parameters => {
 103         additionalProperties => 0,
 104         properties => {
 105             clustername => {
 106                 description => "The name of the cluster.",
 107                 type => 'string', format => 'pve-node',
 108                 maxLength => 15,
 109             },
 110             nodeid => {
 111                 type => 'integer',
 112                 description => "Node id for this node.",
 113                 minimum => 1,
 114                 optional => 1,
 115             },
 116             votes => {
 117                 type => 'integer',
 118                 description => "Number of votes for this node.",
 119                 minimum => 1,
 120                 optional => 1,
 121             },
 122             bindnet0_addr => {
 123                 type => 'string', format => 'ip',
 124                 description => "This specifies the network address the corosync ring 0".
 125                     " executive should bind to and defaults to the local IP address of the node.",
 126                 optional => 1,
 127             },
 128             ring0_addr => {
 129                 type => 'string', format => 'address',
 130                 description => "Hostname (or IP) of the corosync ring0 address of this node.".
 131                     " Defaults to the hostname of the node.",
 132                 optional => 1,
 133             },
 134             bindnet1_addr => {
 135                 type => 'string', format => 'ip',
 136                 description => "This specifies the network address the corosync ring 1".
 137                     " executive should bind to and is optional.",
 138                 optional => 1,
 139             },
 140             ring1_addr => {
 141                 type => 'string', format => 'address',
 142                 description => "Hostname (or IP) of the corosync ring1 address, this".
 143                     " needs an valid bindnet1_addr.",
 144                 optional => 1,
 145             },
 146         },
 147     },
 148     returns => { type => 'null' },
 149
 150     code => sub {
 151         my ($param) = @_;
 152
 153         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
 154
 155         PVE::Cluster::setup_sshd_config(1);
 156         PVE::Cluster::setup_rootsshconfig();
 157         PVE::Cluster::setup_ssh_keys();
 158
 159         -f $authfile || __PACKAGE__->keygen({filename => $authfile});
 160
 161         -f $authfile || die "no authentication key available\n";
 162
 163         my $clustername = $param->{clustername};
 164
 165         $param->{nodeid} = 1 if !$param->{nodeid};
 166
 167         $param->{votes} = 1 if !defined($param->{votes});
 168
 169         my $nodename = PVE::INotify::nodename();
 170
 171         my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 172
 173         $param->{bindnet0_addr} = $local_ip_address
 174             if !defined($param->{bindnet0_addr});
 175
 176         $param->{ring0_addr} = $nodename if !defined($param->{ring0_addr});
 177
 178         die "Param bindnet1_addr and ring1_addr are dependend, use both or none!\n"
 179             if (defined($param->{bindnet1_addr}) != defined($param->{ring1_addr}));
 180
 181         my $bind_is_ipv6 = Net::IP::ip_is_ipv6($param->{bindnet0_addr});
 182
 183         # use string as here-doc format distracts more
 184         my $interfaces = "interface {\n    ringnumber: 0\n" .
 185             "    bindnetaddr: $param->{bindnet0_addr}\n  }";
 186
 187         my $ring_addresses = "ring0_addr: $param->{ring0_addr}" ;
 188
 189         # allow use of multiple rings (rrp) at cluster creation time
 190         if ($param->{bindnet1_addr}) {
 191             die "IPv6 and IPv4 cannot be mixed, use one or the other!\n"
 192                 if Net::IP::ip_is_ipv6($param->{bindnet1_addr}) != $bind_is_ipv6;
 193
 194             $interfaces .= "\n  interface {\n    ringnumber: 1\n" .
 195                 "    bindnetaddr: $param->{bindnet1_addr}\n  }\n";
 196
 197             $interfaces .= "rrp_mode: passive\n"; # only passive is stable and tested
 198
 199             $ring_addresses .= "\n    ring1_addr: $param->{ring1_addr}";
 200         }
 201
 202         # No, corosync cannot deduce this on its own
 203         my $ipversion = $bind_is_ipv6 ? 'ipv6' : 'ipv4';
 204
 205         my $config = <<_EOD;
 206 totem {
 207   version: 2
 208   secauth: on
 209   cluster_name: $clustername
 210   config_version: 1
 211   ip_version: $ipversion
 212   $interfaces
 213 }
 214
 215 nodelist {
 216   node {
 217     $ring_addresses
 218     name: $nodename
 219     nodeid: $param->{nodeid}
 220     quorum_votes: $param->{votes}
 221   }
 222 }
 223
 224 quorum {
 225   provider: corosync_votequorum
 226 }
 227
 228 logging {
 229   to_syslog: yes
 230   debug: off
 231 }
 232 _EOD
 233 ;
 234         PVE::Tools::file_set_contents($clusterconf, $config);
 235
 236         PVE::Cluster::ssh_merge_keys();
 237
 238         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 239
 240         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 241
 242         run_command('systemctl restart pve-cluster'); # restart
 243
 244         run_command('systemctl restart corosync'); # restart
 245
 246         return undef;
 247 }});
 248
 249 __PACKAGE__->register_method ({
 250     name => 'add',
 251     path => 'add',
 252     method => 'PUT',
 253     description => "Adds the current node to an existing cluster.",
 254     parameters => {
 255         additionalProperties => 0,
 256         properties => {
 257             hostname => {
 258                 type => 'string',
 259                 description => "Hostname (or IP) of an existing cluster member."
 260             },
 261             nodeid => {
 262                 type => 'integer',
 263                 description => "Node id for this node.",
 264                 minimum => 1,
 265                 optional => 1,
 266             },
 267             votes => {
 268                 type => 'integer',
 269                 description => "Number of votes for this node",
 270                 minimum => 0,
 271                 optional => 1,
 272             },
 273             force => {
 274                 type => 'boolean',
 275                 description => "Do not throw error if node already exists.",
 276                 optional => 1,
 277             },
 278             ring0_addr => {
 279                 type => 'string', format => 'address',
 280                 description => "Hostname (or IP) of the corosync ring0 address of this node.".
 281                     " Defaults to nodes hostname.",
 282                 optional => 1,
 283             },
 284             ring1_addr => {
 285                 type => 'string', format => 'address',
 286                 description => "Hostname (or IP) of the corosync ring1 address, this".
 287                     " needs an valid configured ring 1 interface in the cluster.",
 288                 optional => 1,
 289             },
 290         },
 291     },
 292     returns => { type => 'null' },
 293
 294     code => sub {
 295         my ($param) = @_;
 296
 297         my $nodename = PVE::INotify::nodename();
 298
 299         PVE::Cluster::setup_sshd_config();
 300         PVE::Cluster::setup_rootsshconfig();
 301         PVE::Cluster::setup_ssh_keys();
 302
 303         PVE::Cluster::assert_joinable($param->{ring0_addr}, $param->{ring1_addr}, $param->{force});
 304
 305         my $host = $param->{hostname};
 306
 307         # make sure known_hosts is on local filesystem
 308         PVE::Cluster::ssh_unmerge_known_hosts();
 309
 310         my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"];
 311         run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
 312                                 'errmsg' => "unable to copy ssh ID");
 313
 314         $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
 315                 'pvecm', 'addnode', $nodename, '--force', 1];
 316
 317         push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
 318         push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
 319         push @$cmd, '--ring0_addr', $param->{ring0_addr} if defined($param->{ring0_addr});
 320         push @$cmd, '--ring1_addr', $param->{ring1_addr} if defined($param->{ring1_addr});
 321
 322         if (system (@$cmd) != 0) {
 323             my $cmdtxt = join (' ', @$cmd);
 324             die "unable to add node: command failed ($cmdtxt)\n";
 325         }
 326
 327         my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
 328         mkdir $tmpdir;
 329
 330         eval {
 331             print "copy corosync auth key\n";
 332             $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
 333                     "[$host]:$authfile $clusterconf", $tmpdir];
 334
 335             system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
 336
 337             mkdir "/etc/corosync";
 338             my $confbase = basename($clusterconf);
 339
 340             $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'";
 341             system($cmd) == 0 || die "can't copy cluster configuration\n";
 342
 343             my $keybase = basename($authfile);
 344             system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
 345                 die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
 346
 347             print "stopping pve-cluster service\n";
 348
 349             system("umount $basedir -f >/dev/null 2>&1");
 350             system("systemctl stop pve-cluster") == 0 ||
 351                 die "can't stop pve-cluster service\n";
 352
 353             backup_database();
 354
 355             unlink $dbfile;
 356
 357             system("systemctl start pve-cluster") == 0 ||
 358                 die "starting pve-cluster failed\n";
 359
 360             system("systemctl start corosync");
 361
 362             # wait for quorum
 363             my $printqmsg = 1;
 364             while (!PVE::Cluster::check_cfs_quorum(1)) {
 365                 if ($printqmsg) {
 366                     print "waiting for quorum...";
 367                     STDOUT->flush();
 368                     $printqmsg = 0;
 369                 }
 370                 sleep(1);
 371             }
 372             print "OK\n" if !$printqmsg;
 373
 374             my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 375
 376             print "generating node certificates\n";
 377             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 378
 379             print "merge known_hosts file\n";
 380             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 381
 382             print "restart services\n";
 383             # restart pvedaemon (changed certs)
 384             system("systemctl restart pvedaemon");
 385             # restart pveproxy (changed certs)
 386             system("systemctl restart pveproxy");
 387
 388             print "successfully added node '$nodename' to cluster.\n";
 389         };
 390         my $err = $@;
 391
 392         rmtree $tmpdir;
 393
 394         die $err if $err;
 395
 396         return undef;
 397     }});
 398
 399 __PACKAGE__->register_method ({
 400     name => 'status',
 401     path => 'status',
 402     method => 'GET',
 403     description => "Displays the local view of the cluster status.",
 404     parameters => {
 405         additionalProperties => 0,
 406         properties => {},
 407     },
 408     returns => { type => 'null' },
 409
 410     code => sub {
 411         my ($param) = @_;
 412
 413         PVE::Corosync::check_conf_exists();
 414
 415         my $cmd = ['corosync-quorumtool', '-siH'];
 416
 417         exec (@$cmd);
 418
 419         exit (-1); # should not be reached
 420     }});
 421
 422 __PACKAGE__->register_method ({
 423     name => 'nodes',
 424     path => 'nodes',
 425     method => 'GET',
 426     description => "Displays the local view of the cluster nodes.",
 427     parameters => {
 428         additionalProperties => 0,
 429         properties => {},
 430     },
 431     returns => { type => 'null' },
 432
 433     code => sub {
 434         my ($param) = @_;
 435
 436         PVE::Corosync::check_conf_exists();
 437
 438         my $cmd = ['corosync-quorumtool', '-l'];
 439
 440         exec (@$cmd);
 441
 442         exit (-1); # should not be reached
 443     }});
 444
 445 __PACKAGE__->register_method ({
 446     name => 'expected',
 447     path => 'expected',
 448     method => 'PUT',
 449     description => "Tells corosync a new value of expected votes.",
 450     parameters => {
 451         additionalProperties => 0,
 452         properties => {
 453             expected => {
 454                 type => 'integer',
 455                 description => "Expected votes",
 456                 minimum => 1,
 457             },
 458         },
 459     },
 460     returns => { type => 'null' },
 461
 462     code => sub {
 463         my ($param) = @_;
 464
 465         PVE::Corosync::check_conf_exists();
 466
 467         my $cmd = ['corosync-quorumtool', '-e', $param->{expected}];
 468
 469         exec (@$cmd);
 470
 471         exit (-1); # should not be reached
 472
 473     }});
 474
 475 __PACKAGE__->register_method ({
 476     name => 'updatecerts',
 477     path => 'updatecerts',
 478     method => 'PUT',
 479     description => "Update node certificates (and generate all needed files/directories).",
 480     parameters => {
 481         additionalProperties => 0,
 482         properties => {
 483             force => {
 484                 description => "Force generation of new SSL certifate.",
 485                 type => 'boolean',
 486                 optional => 1,
 487             },
 488             silent => {
 489                 description => "Ignore errors (i.e. when cluster has no quorum).",
 490                 type => 'boolean',
 491                 optional => 1,
 492             },
 493         },
 494     },
 495     returns => { type => 'null' },
 496     code => sub {
 497         my ($param) = @_;
 498
 499         PVE::Cluster::setup_rootsshconfig();
 500
 501         PVE::Cluster::gen_pve_vzdump_symlink();
 502
 503         if (!PVE::Cluster::check_cfs_quorum(1)) {
 504             return undef if $param->{silent};
 505             die "no quorum - unable to update files\n";
 506         }
 507
 508         PVE::Cluster::setup_ssh_keys();
 509
 510         my $nodename = PVE::INotify::nodename();
 511
 512         my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 513
 514         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
 515         PVE::Cluster::ssh_merge_keys();
 516         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
 517         PVE::Cluster::gen_pve_vzdump_files();
 518
 519         return undef;
 520     }});
 521
 522 __PACKAGE__->register_method ({
 523     name => 'mtunnel',
 524     path => 'mtunnel',
 525     method => 'POST',
 526     description => "Used by VM/CT migration - do not use manually.",
 527     parameters => {
 528         additionalProperties => 0,
 529         properties => {
 530             get_migration_ip => {
 531                 type => 'boolean',
 532                 default => 0,
 533                 description => 'return the migration IP, if configured',
 534                 optional => 1,
 535             },
 536             migration_network => {
 537                 type => 'string',
 538                 format => 'CIDR',
 539                 description => 'the migration network used to detect the local migration IP',
 540                 optional => 1,
 541             },
 542             'run-command' => {
 543                 type => 'boolean',
 544                 description => 'Run a command with a tcp socket as standard input.'
 545                               .' The IP address and port are printed via this'
 546                               ." command's stdandard output first, each on a separate line.",
 547                 optional => 1,
 548             },
 549             'extra-args' => PVE::JSONSchema::get_standard_option('extra-args'),
 550         },
 551     },
 552     returns => { type => 'null'},
 553     code => sub {
 554         my ($param) = @_;
 555
 556         if (!PVE::Cluster::check_cfs_quorum(1)) {
 557             print "no quorum\n";
 558             return undef;
 559         }
 560
 561         my $network = $param->{migration_network};
 562         if ($param->{get_migration_ip}) {
 563             die "cannot use --run-command with --get_migration_ip\n"
 564                 if $param->{'run-command'};
 565             if (my $ip = PVE::Cluster::get_local_migration_ip($network)) {
 566                 print "ip: '$ip'\n";
 567             } else {
 568                 print "no ip\n";
 569             }
 570             # do not keep tunnel open when asked for migration ip
 571             return undef;
 572         }
 573
 574         if ($param->{'run-command'}) {
 575             my $cmd = $param->{'extra-args'};
 576             die "missing command\n"
 577                 if !$cmd || !scalar(@$cmd);
 578
 579             # Get an ip address to listen on, and find a free migration port
 580             my ($ip, $family);
 581             if (defined($network)) {
 582                 $ip = PVE::Cluster::get_local_migration_ip($network)
 583                     or die "failed to get migration IP address to listen on\n";
 584                 $family = PVE::Tools::get_host_address_family($ip);
 585             } else {
 586                 my $nodename = PVE::INotify::nodename();
 587                 ($ip, $family) = PVE::Network::get_ip_from_hostname($nodename, 0);
 588             }
 589             my $port = PVE::Tools::next_migrate_port($family, $ip);
 590
 591             PVE::Tools::pipe_socket_to_command($cmd, $ip, $port);
 592             return undef;
 593         }
 594
 595         print "tunnel online\n";
 596         *STDOUT->flush();
 597
 598         while (my $line = <STDIN>) {
 599             chomp $line;
 600             last if $line =~ m/^quit$/;
 601         }
 602
 603         return undef;
 604     }});
 605
 606
 607 our $cmddef = {
 608     keygen => [ __PACKAGE__, 'keygen', ['filename']],
 609     create => [ __PACKAGE__, 'create', ['clustername']],
 610     add => [ __PACKAGE__, 'add', ['hostname']],
 611     addnode => [ 'PVE::API2::ClusterConfig', 'addnode', ['node']],
 612     delnode => [ 'PVE::API2::ClusterConfig', 'delnode', ['node']],
 613     status => [ __PACKAGE__, 'status' ],
 614     nodes => [ __PACKAGE__, 'nodes' ],
 615     expected => [ __PACKAGE__, 'expected', ['expected']],
 616     updatecerts => [ __PACKAGE__, 'updatecerts', []],
 617     mtunnel => [ __PACKAGE__, 'mtunnel', ['extra-args']],
 618 };
 619
 620 1;