data/PVE/CLI/pvecm.pm

   1 package PVE::CLI::pvecm;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Net::IP;
   7 use File::Path;
   8 use File::Basename;
   9 use PVE::Tools qw(run_command);
  10 use PVE::Cluster;
  11 use PVE::INotify;
  12 use PVE::JSONSchema;
  13 use PVE::CLIHandler;
  14 use PVE::API2::ClusterConfig;
  15 use PVE::Corosync;
  16
  17 use base qw(PVE::CLIHandler);
  18
  19 $ENV{HOME} = '/root'; # for ssh-copy-id
  20
  21 my $basedir = "/etc/pve";
  22 my $clusterconf = "$basedir/corosync.conf";
  23 my $libdir = "/var/lib/pve-cluster";
  24 my $backupdir = "/var/lib/pve-cluster/backup";
  25 my $dbfile = "$libdir/config.db";
  26 my $authfile = "/etc/corosync/authkey";
  27
  28 sub backup_database {
  29
  30     print "backup old database\n";
  31
  32     mkdir $backupdir;
  33
  34     my $ctime = time();
  35     my $cmd = [
  36         ['echo', '.dump'],
  37         ['sqlite3', $dbfile],
  38         ['gzip', '-', \ ">${backupdir}/config-${ctime}.sql.gz"],
  39     ];
  40
  41     run_command($cmd, 'errmsg' => "cannot backup old database\n");
  42
  43     # purge older backup
  44     my $maxfiles = 10;
  45
  46     my @bklist = ();
  47     foreach my $fn (<$backupdir/config-*.sql.gz>) {
  48         if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
  49             push @bklist, [$fn, $1];
  50         }
  51     }
  52
  53     @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
  54
  55     while (scalar (@bklist) >= $maxfiles) {
  56         my $d = pop @bklist;
  57         print "delete old backup '$d->[0]'\n";
  58         unlink $d->[0];
  59     }
  60 }
  61
  62
  63 __PACKAGE__->register_method ({
  64     name => 'keygen',
  65     path => 'keygen',
  66     method => 'PUT',
  67     description => "Generate new cryptographic key for corosync.",
  68     parameters => {
  69         additionalProperties => 0,
  70         properties => {
  71             filename => {
  72                 type => 'string',
  73                 description => "Output file name"
  74             }
  75         },
  76     },
  77     returns => { type => 'null' },
  78
  79     code => sub {
  80         my ($param) = @_;
  81
  82         my $filename = $param->{filename};
  83
  84         # test EUID
  85         $> == 0 || die "Error: Authorization key must be generated as root user.\n";
  86         my $dirname = dirname($filename);
  87
  88         die "key file '$filename' already exists\n" if -e $filename;
  89
  90         File::Path::make_path($dirname) if $dirname;
  91
  92         run_command(['corosync-keygen', '-l', '-k', $filename]);
  93
  94         return undef;
  95     }});
  96
  97 __PACKAGE__->register_method ({
  98     name => 'create',
  99     path => 'create',
 100     method => 'PUT',
 101     description => "Generate new cluster configuration.",
 102     parameters => {
 103         additionalProperties => 0,
 104         properties => {
 105             clustername => {
 106                 description => "The name of the cluster.",
 107                 type => 'string', format => 'pve-node',
 108                 maxLength => 15,
 109             },
 110             nodeid => {
 111                 type => 'integer',
 112                 description => "Node id for this node.",
 113                 minimum => 1,
 114                 optional => 1,
 115             },
 116             votes => {
 117                 type => 'integer',
 118                 description => "Number of votes for this node.",
 119                 minimum => 1,
 120                 optional => 1,
 121             },
 122             bindnet0_addr => {
 123                 type => 'string', format => 'ip',
 124                 description => "This specifies the network address the corosync ring 0".
 125                     " executive should bind to and defaults to the local IP address of the node.",
 126                 optional => 1,
 127             },
 128             ring0_addr => {
 129                 type => 'string', format => 'address',
 130                 description => "Hostname (or IP) of the corosync ring0 address of this node.".
 131                     " Defaults to the hostname of the node.",
 132                 optional => 1,
 133             },
 134             bindnet1_addr => {
 135                 type => 'string', format => 'ip',
 136                 description => "This specifies the network address the corosync ring 1".
 137                     " executive should bind to and is optional.",
 138                 optional => 1,
 139             },
 140             ring1_addr => {
 141                 type => 'string', format => 'address',
 142                 description => "Hostname (or IP) of the corosync ring1 address, this".
 143                     " needs an valid bindnet1_addr.",
 144                 optional => 1,
 145             },
 146         },
 147     },
 148     returns => { type => 'null' },
 149
 150     code => sub {
 151         my ($param) = @_;
 152
 153         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
 154
 155         PVE::Cluster::setup_sshd_config(1);
 156         PVE::Cluster::setup_rootsshconfig();
 157         PVE::Cluster::setup_ssh_keys();
 158
 159         -f $authfile || __PACKAGE__->keygen({filename => $authfile});
 160
 161         -f $authfile || die "no authentication key available\n";
 162
 163         my $clustername = $param->{clustername};
 164
 165         $param->{nodeid} = 1 if !$param->{nodeid};
 166
 167         $param->{votes} = 1 if !defined($param->{votes});
 168
 169         my $nodename = PVE::INotify::nodename();
 170
 171         my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 172
 173         $param->{bindnet0_addr} = $local_ip_address
 174             if !defined($param->{bindnet0_addr});
 175
 176         $param->{ring0_addr} = $nodename if !defined($param->{ring0_addr});
 177
 178         die "Param bindnet1_addr and ring1_addr are dependend, use both or none!\n"
 179             if (defined($param->{bindnet1_addr}) != defined($param->{ring1_addr}));
 180
 181         my $bind_is_ipv6 = Net::IP::ip_is_ipv6($param->{bindnet0_addr});
 182
 183         # use string as here-doc format distracts more
 184         my $interfaces = "interface {\n    ringnumber: 0\n" .
 185             "    bindnetaddr: $param->{bindnet0_addr}\n  }";
 186
 187         my $ring_addresses = "ring0_addr: $param->{ring0_addr}" ;
 188
 189         # allow use of multiple rings (rrp) at cluster creation time
 190         if ($param->{bindnet1_addr}) {
 191             die "IPv6 and IPv4 cannot be mixed, use one or the other!\n"
 192                 if Net::IP::ip_is_ipv6($param->{bindnet1_addr}) != $bind_is_ipv6;
 193
 194             $interfaces .= "\n  interface {\n    ringnumber: 1\n" .
 195                 "    bindnetaddr: $param->{bindnet1_addr}\n  }\n";
 196
 197             $interfaces .= "rrp_mode: passive\n"; # only passive is stable and tested
 198
 199             $ring_addresses .= "\n    ring1_addr: $param->{ring1_addr}";
 200         }
 201
 202         # No, corosync cannot deduce this on its own
 203         my $ipversion = $bind_is_ipv6 ? 'ipv6' : 'ipv4';
 204
 205         my $config = <<_EOD;
 206 totem {
 207   version: 2
 208   secauth: on
 209   cluster_name: $clustername
 210   config_version: 1
 211   ip_version: $ipversion
 212   $interfaces
 213 }
 214
 215 nodelist {
 216   node {
 217     $ring_addresses
 218     name: $nodename
 219     nodeid: $param->{nodeid}
 220     quorum_votes: $param->{votes}
 221   }
 222 }
 223
 224 quorum {
 225   provider: corosync_votequorum
 226 }
 227
 228 logging {
 229   to_syslog: yes
 230   debug: off
 231 }
 232 _EOD
 233 ;
 234         PVE::Tools::file_set_contents($clusterconf, $config);
 235
 236         PVE::Cluster::ssh_merge_keys();
 237
 238         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 239
 240         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 241
 242         run_command('systemctl restart pve-cluster'); # restart
 243
 244         run_command('systemctl restart corosync'); # restart
 245
 246         return undef;
 247 }});
 248
 249 __PACKAGE__->register_method ({
 250     name => 'add',
 251     path => 'add',
 252     method => 'PUT',
 253     description => "Adds the current node to an existing cluster.",
 254     parameters => {
 255         additionalProperties => 0,
 256         properties => {
 257             hostname => {
 258                 type => 'string',
 259                 description => "Hostname (or IP) of an existing cluster member."
 260             },
 261             nodeid => {
 262                 type => 'integer',
 263                 description => "Node id for this node.",
 264                 minimum => 1,
 265                 optional => 1,
 266             },
 267             votes => {
 268                 type => 'integer',
 269                 description => "Number of votes for this node",
 270                 minimum => 0,
 271                 optional => 1,
 272             },
 273             force => {
 274                 type => 'boolean',
 275                 description => "Do not throw error if node already exists.",
 276                 optional => 1,
 277             },
 278             ring0_addr => {
 279                 type => 'string', format => 'address',
 280                 description => "Hostname (or IP) of the corosync ring0 address of this node.".
 281                     " Defaults to nodes hostname.",
 282                 optional => 1,
 283             },
 284             ring1_addr => {
 285                 type => 'string', format => 'address',
 286                 description => "Hostname (or IP) of the corosync ring1 address, this".
 287                     " needs an valid configured ring 1 interface in the cluster.",
 288                 optional => 1,
 289             },
 290         },
 291     },
 292     returns => { type => 'null' },
 293
 294     code => sub {
 295         my ($param) = @_;
 296
 297         my $nodename = PVE::INotify::nodename();
 298
 299         PVE::Cluster::setup_sshd_config();
 300         PVE::Cluster::setup_rootsshconfig();
 301         PVE::Cluster::setup_ssh_keys();
 302
 303         my $host = $param->{hostname};
 304
 305         my ($errors, $warnings) = ('', '');
 306
 307         my $error = sub {
 308             my ($msg, $suppress) = @_;
 309
 310             if ($suppress) {
 311                 $warnings .= "* $msg\n";
 312             } else {
 313                 $errors .= "* $msg\n";
 314             }
 315         };
 316
 317         if (!$param->{force}) {
 318
 319             if (-f $authfile) {
 320                 &$error("authentication key '$authfile' already exists", $param->{force});
 321             }
 322
 323             if (-f $clusterconf)  {
 324                 &$error("cluster config '$clusterconf' already exists", $param->{force});
 325             }
 326
 327             my $vmlist = PVE::Cluster::get_vmlist();
 328             if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 329                 &$error("this host already contains virtual guests", $param->{force});
 330             }
 331
 332             if (system("corosync-quorumtool -l >/dev/null 2>&1") == 0) {
 333                 &$error("corosync is already running, is this node already in a cluster?!", $param->{force});
 334             }
 335         }
 336
 337         # check if corosync ring IPs are configured on the current nodes interfaces
 338         my $check_ip = sub {
 339             my $ip = shift;
 340             if (defined($ip)) {
 341                 if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
 342                     my $host = $ip;
 343                     eval { $ip = PVE::Network::get_ip_from_hostname($host); };
 344                     if ($@) {
 345                         &$error("cannot use '$host': $@\n") ;
 346                         return;
 347                     }
 348                 }
 349
 350                 my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
 351                 my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
 352
 353                 &$error("cannot use IP '$ip', it must be configured exactly once on local node!\n")
 354                     if (scalar(@$configured_ips) != 1);
 355             }
 356         };
 357
 358         &$check_ip($param->{ring0_addr});
 359         &$check_ip($param->{ring1_addr});
 360
 361         warn "warning, ignore the following errors:\n$warnings" if $warnings;
 362         die "detected the following error(s):\n$errors" if $errors;
 363
 364         # make sure known_hosts is on local filesystem
 365         PVE::Cluster::ssh_unmerge_known_hosts();
 366
 367         my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"];
 368         run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
 369                                 'errmsg' => "unable to copy ssh ID");
 370
 371         $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
 372                 'pvecm', 'addnode', $nodename, '--force', 1];
 373
 374         push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
 375
 376         push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
 377
 378         push @$cmd, '--ring0_addr', $param->{ring0_addr} if defined($param->{ring0_addr});
 379
 380         push @$cmd, '--ring1_addr', $param->{ring1_addr} if defined($param->{ring1_addr});
 381
 382         if (system (@$cmd) != 0) {
 383             my $cmdtxt = join (' ', @$cmd);
 384             die "unable to add node: command failed ($cmdtxt)\n";
 385         }
 386
 387         my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
 388         mkdir $tmpdir;
 389
 390         eval {
 391             print "copy corosync auth key\n";
 392             $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
 393                     "[$host]:$authfile $clusterconf", $tmpdir];
 394
 395             system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
 396
 397             mkdir "/etc/corosync";
 398             my $confbase = basename($clusterconf);
 399
 400             $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'";
 401             system($cmd) == 0 || die "can't copy cluster configuration\n";
 402
 403             my $keybase = basename($authfile);
 404             system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
 405                 die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
 406
 407             print "stopping pve-cluster service\n";
 408
 409             system("umount $basedir -f >/dev/null 2>&1");
 410             system("systemctl stop pve-cluster") == 0 ||
 411                 die "can't stop pve-cluster service\n";
 412
 413             backup_database();
 414
 415             unlink $dbfile;
 416
 417             system("systemctl start pve-cluster") == 0 ||
 418                 die "starting pve-cluster failed\n";
 419
 420             system("systemctl start corosync");
 421
 422             # wait for quorum
 423             my $printqmsg = 1;
 424             while (!PVE::Cluster::check_cfs_quorum(1)) {
 425                 if ($printqmsg) {
 426                     print "waiting for quorum...";
 427                     STDOUT->flush();
 428                     $printqmsg = 0;
 429                 }
 430                 sleep(1);
 431             }
 432             print "OK\n" if !$printqmsg;
 433
 434             my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 435
 436             print "generating node certificates\n";
 437             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 438
 439             print "merge known_hosts file\n";
 440             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 441
 442             print "restart services\n";
 443             # restart pvedaemon (changed certs)
 444             system("systemctl restart pvedaemon");
 445             # restart pveproxy (changed certs)
 446             system("systemctl restart pveproxy");
 447
 448             print "successfully added node '$nodename' to cluster.\n";
 449         };
 450         my $err = $@;
 451
 452         rmtree $tmpdir;
 453
 454         die $err if $err;
 455
 456         return undef;
 457     }});
 458
 459 __PACKAGE__->register_method ({
 460     name => 'status',
 461     path => 'status',
 462     method => 'GET',
 463     description => "Displays the local view of the cluster status.",
 464     parameters => {
 465         additionalProperties => 0,
 466         properties => {},
 467     },
 468     returns => { type => 'null' },
 469
 470     code => sub {
 471         my ($param) = @_;
 472
 473         PVE::Corosync::check_conf_exists();
 474
 475         my $cmd = ['corosync-quorumtool', '-siH'];
 476
 477         exec (@$cmd);
 478
 479         exit (-1); # should not be reached
 480     }});
 481
 482 __PACKAGE__->register_method ({
 483     name => 'nodes',
 484     path => 'nodes',
 485     method => 'GET',
 486     description => "Displays the local view of the cluster nodes.",
 487     parameters => {
 488         additionalProperties => 0,
 489         properties => {},
 490     },
 491     returns => { type => 'null' },
 492
 493     code => sub {
 494         my ($param) = @_;
 495
 496         PVE::Corosync::check_conf_exists();
 497
 498         my $cmd = ['corosync-quorumtool', '-l'];
 499
 500         exec (@$cmd);
 501
 502         exit (-1); # should not be reached
 503     }});
 504
 505 __PACKAGE__->register_method ({
 506     name => 'expected',
 507     path => 'expected',
 508     method => 'PUT',
 509     description => "Tells corosync a new value of expected votes.",
 510     parameters => {
 511         additionalProperties => 0,
 512         properties => {
 513             expected => {
 514                 type => 'integer',
 515                 description => "Expected votes",
 516                 minimum => 1,
 517             },
 518         },
 519     },
 520     returns => { type => 'null' },
 521
 522     code => sub {
 523         my ($param) = @_;
 524
 525         PVE::Corosync::check_conf_exists();
 526
 527         my $cmd = ['corosync-quorumtool', '-e', $param->{expected}];
 528
 529         exec (@$cmd);
 530
 531         exit (-1); # should not be reached
 532
 533     }});
 534
 535 __PACKAGE__->register_method ({
 536     name => 'updatecerts',
 537     path => 'updatecerts',
 538     method => 'PUT',
 539     description => "Update node certificates (and generate all needed files/directories).",
 540     parameters => {
 541         additionalProperties => 0,
 542         properties => {
 543             force => {
 544                 description => "Force generation of new SSL certifate.",
 545                 type => 'boolean',
 546                 optional => 1,
 547             },
 548             silent => {
 549                 description => "Ignore errors (i.e. when cluster has no quorum).",
 550                 type => 'boolean',
 551                 optional => 1,
 552             },
 553         },
 554     },
 555     returns => { type => 'null' },
 556     code => sub {
 557         my ($param) = @_;
 558
 559         PVE::Cluster::setup_rootsshconfig();
 560
 561         PVE::Cluster::gen_pve_vzdump_symlink();
 562
 563         if (!PVE::Cluster::check_cfs_quorum(1)) {
 564             return undef if $param->{silent};
 565             die "no quorum - unable to update files\n";
 566         }
 567
 568         PVE::Cluster::setup_ssh_keys();
 569
 570         my $nodename = PVE::INotify::nodename();
 571
 572         my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 573
 574         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
 575         PVE::Cluster::ssh_merge_keys();
 576         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
 577         PVE::Cluster::gen_pve_vzdump_files();
 578
 579         return undef;
 580     }});
 581
 582 __PACKAGE__->register_method ({
 583     name => 'mtunnel',
 584     path => 'mtunnel',
 585     method => 'POST',
 586     description => "Used by VM/CT migration - do not use manually.",
 587     parameters => {
 588         additionalProperties => 0,
 589         properties => {
 590             get_migration_ip => {
 591                 type => 'boolean',
 592                 default => 0,
 593                 description => 'return the migration IP, if configured',
 594                 optional => 1,
 595             },
 596             migration_network => {
 597                 type => 'string',
 598                 format => 'CIDR',
 599                 description => 'the migration network used to detect the local migration IP',
 600                 optional => 1,
 601             },
 602             'run-command' => {
 603                 type => 'boolean',
 604                 description => 'Run a command with a tcp socket as standard input.'
 605                               .' The IP address and port are printed via this'
 606                               ." command's stdandard output first, each on a separate line.",
 607                 optional => 1,
 608             },
 609             'extra-args' => PVE::JSONSchema::get_standard_option('extra-args'),
 610         },
 611     },
 612     returns => { type => 'null'},
 613     code => sub {
 614         my ($param) = @_;
 615
 616         if (!PVE::Cluster::check_cfs_quorum(1)) {
 617             print "no quorum\n";
 618             return undef;
 619         }
 620
 621         my $network = $param->{migration_network};
 622         if ($param->{get_migration_ip}) {
 623             die "cannot use --run-command with --get_migration_ip\n"
 624                 if $param->{'run-command'};
 625             if (my $ip = PVE::Cluster::get_local_migration_ip($network)) {
 626                 print "ip: '$ip'\n";
 627             } else {
 628                 print "no ip\n";
 629             }
 630             # do not keep tunnel open when asked for migration ip
 631             return undef;
 632         }
 633
 634         if ($param->{'run-command'}) {
 635             my $cmd = $param->{'extra-args'};
 636             die "missing command\n"
 637                 if !$cmd || !scalar(@$cmd);
 638
 639             # Get an ip address to listen on, and find a free migration port
 640             my ($ip, $family);
 641             if (defined($network)) {
 642                 $ip = PVE::Cluster::get_local_migration_ip($network)
 643                     or die "failed to get migration IP address to listen on\n";
 644                 $family = PVE::Tools::get_host_address_family($ip);
 645             } else {
 646                 my $nodename = PVE::INotify::nodename();
 647                 ($ip, $family) = PVE::Network::get_ip_from_hostname($nodename, 0);
 648             }
 649             my $port = PVE::Tools::next_migrate_port($family, $ip);
 650
 651             PVE::Tools::pipe_socket_to_command($cmd, $ip, $port);
 652             return undef;
 653         }
 654
 655         print "tunnel online\n";
 656         *STDOUT->flush();
 657
 658         while (my $line = <STDIN>) {
 659             chomp $line;
 660             last if $line =~ m/^quit$/;
 661         }
 662
 663         return undef;
 664     }});
 665
 666
 667 our $cmddef = {
 668     keygen => [ __PACKAGE__, 'keygen', ['filename']],
 669     create => [ __PACKAGE__, 'create', ['clustername']],
 670     add => [ __PACKAGE__, 'add', ['hostname']],
 671     addnode => [ 'PVE::API2::ClusterConfig', 'addnode', ['node']],
 672     delnode => [ 'PVE::API2::ClusterConfig', 'delnode', ['node']],
 673     status => [ __PACKAGE__, 'status' ],
 674     nodes => [ __PACKAGE__, 'nodes' ],
 675     expected => [ __PACKAGE__, 'expected', ['expected']],
 676     updatecerts => [ __PACKAGE__, 'updatecerts', []],
 677     mtunnel => [ __PACKAGE__, 'mtunnel', ['extra-args']],
 678 };
 679
 680 1;