data/PVE/pvecm

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5 use Getopt::Long;
   6 use Socket;
   7 use IO::File;
   8 use Net::IP;
   9 use File::Path;
  10 use File::Basename;
  11 use Data::Dumper; # fixme: remove
  12 use PVE::Tools;
  13 use PVE::Cluster;
  14 use PVE::INotify;
  15 use PVE::JSONSchema;
  16 use PVE::CLIHandler;
  17
  18 use base qw(PVE::CLIHandler);
  19
  20 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
  21 $ENV{LC_ALL} = 'C';
  22 $ENV{HOME} = '/root'; # for ssh-copy-id
  23
  24 my ($local_ip_address, $nodename);
  25
  26 if (defined($ARGV[0]) && $ARGV[0] ne 'printmanpod' && $ARGV[0] ne 'verifyapi') {
  27     die "please run as root\n" if $> != 0;
  28
  29     $nodename = PVE::INotify::nodename();
  30     # trigger check that we have resolvable name
  31     $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
  32 }
  33
  34 my $basedir = "/etc/pve";
  35 my $clusterconf = "$basedir/corosync.conf";
  36 my $libdir = "/var/lib/pve-cluster";
  37 my $backupdir = "/var/lib/pve-cluster/backup";
  38 my $dbfile = "$libdir/config.db";
  39 my $authfile = "/etc/corosync/authkey";
  40
  41
  42 sub backup_database {
  43
  44     print "backup old database\n";
  45
  46     mkdir $backupdir;
  47
  48     my $ctime = time();
  49     my $cmd = "echo '.dump' |";
  50     $cmd .= "sqlite3 '$dbfile' |";
  51     $cmd .= "gzip - >'${backupdir}/config-${ctime}.sql.gz'";
  52
  53     system($cmd) == 0 ||
  54         die "can't backup old database: $!\n";
  55
  56     # purge older backup
  57     my $maxfiles = 10;
  58
  59     my @bklist = ();
  60     foreach my $fn (<$backupdir/config-*.sql.gz>) {
  61         if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
  62             push @bklist, [$fn, $1];
  63         }
  64     }
  65
  66     @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
  67
  68     while (scalar (@bklist) >= $maxfiles) {
  69         my $d = pop @bklist;
  70         print "delete old backup '$d->[0]'\n";
  71         unlink $d->[0];
  72     }
  73 }
  74
  75 __PACKAGE__->register_method ({
  76     name => 'keygen',
  77     path => 'keygen',
  78     method => 'PUT',
  79     description => "Generate new cryptographic key for corosync.",
  80     parameters => {
  81         additionalProperties => 0,
  82         properties => {
  83             filename => {
  84                 type => 'string',
  85                 description => "Output file name"
  86             }
  87         },
  88     },
  89     returns => { type => 'null' },
  90
  91     code => sub {
  92         my ($param) = @_;
  93
  94         my $filename = $param->{filename};
  95
  96         # test EUID
  97         $> == 0 || die "Error: Authorization key must be generated as root user.\n";
  98         my $dirname = dirname($filename);
  99         my $basename = basename($filename);
 100
 101         die "key file '$filename' already exists\n" if -e $filename;
 102
 103         File::Path::make_path($dirname) if $dirname;
 104
 105         my $cmd = ['corosync-keygen', '-l', '-k', $filename];
 106         PVE::Tools::run_command($cmd);
 107
 108         return undef;
 109     }});
 110
 111 __PACKAGE__->register_method ({
 112     name => 'create',
 113     path => 'create',
 114     method => 'PUT',
 115     description => "Generate new cluster configuration.",
 116     parameters => {
 117         additionalProperties => 0,
 118         properties => {
 119             clustername => {
 120                 description => "The name of the cluster.",
 121                 type => 'string', format => 'pve-node',
 122                 maxLength => 15,
 123             },
 124             nodeid => {
 125                 type => 'integer',
 126                 description => "Node id for this node.",
 127                 minimum => 1,
 128                 optional => 1,
 129             },
 130             votes => {
 131                 type => 'integer',
 132                 description => "Number of votes for this node.",
 133                 minimum => 1,
 134                 optional => 1,
 135             },
 136         },
 137     },
 138     returns => { type => 'null' },
 139
 140     code => sub {
 141         my ($param) = @_;
 142
 143         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
 144
 145         PVE::Cluster::setup_sshd_config();
 146         PVE::Cluster::setup_rootsshconfig();
 147         PVE::Cluster::setup_ssh_keys();
 148
 149         -f $authfile || __PACKAGE__->keygen({filename => $authfile});
 150
 151         -f $authfile || die "no authentication key available\n";
 152
 153         my $clustername = $param->{clustername};
 154
 155         $param->{nodeid} = 1 if !$param->{nodeid};
 156
 157         $param->{votes} = 1 if !defined($param->{votes});
 158
 159         # No, corosync cannot deduce this on its own
 160         my $ipversion = Net::IP::ip_is_ipv6($local_ip_address) ? 'ipv6' : 'ipv4';
 161
 162         my $config = <<_EOD;
 163 totem {
 164   version: 2
 165   secauth: on
 166   cluster_name: $clustername
 167   config_version: 1
 168   ip_version: $ipversion
 169   interface {
 170     ringnumber: 0
 171     bindnetaddr: $local_ip_address
 172   }
 173 }
 174
 175 nodelist {
 176   node {
 177     ring0_addr: $nodename
 178     nodeid: $param->{nodeid}
 179     quorum_votes: $param->{votes}
 180   }
 181 }
 182
 183 quorum {
 184   provider: corosync_votequorum
 185 }
 186
 187 logging {
 188   to_syslog: yes
 189   debug: off
 190 }
 191 _EOD
 192 ;
 193         PVE::Tools::file_set_contents($clusterconf, $config);
 194
 195         PVE::Cluster::ssh_merge_keys();
 196
 197         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 198
 199         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 200
 201         PVE::Tools::run_command('systemctl restart pve-cluster'); # restart
 202
 203         PVE::Tools::run_command('systemctl restart corosync'); # restart
 204
 205         return undef;
 206 }});
 207
 208 __PACKAGE__->register_method ({
 209     name => 'addnode',
 210     path => 'addnode',
 211     method => 'PUT',
 212     description => "Adds a node to the cluster configuration.",
 213     parameters => {
 214         additionalProperties => 0,
 215         properties => {
 216             node => PVE::JSONSchema::get_standard_option('pve-node'),
 217             nodeid => {
 218                 type => 'integer',
 219                 description => "Node id for this node.",
 220                 minimum => 1,
 221                 optional => 1,
 222             },
 223             votes => {
 224                 type => 'integer',
 225                 description => "Number of votes for this node",
 226                 minimum => 0,
 227                 optional => 1,
 228             },
 229             force => {
 230                 type => 'boolean',
 231                 description => "Do not throw error if node already exists.",
 232                 optional => 1,
 233             },
 234         },
 235     },
 236     returns => { type => 'null' },
 237
 238     code => sub {
 239         my ($param) = @_;
 240
 241         PVE::Cluster::check_cfs_quorum();
 242
 243         my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 244
 245         my $nodelist = corosync_nodelist($conf);
 246
 247         my $name = $param->{node};
 248
 249         if (defined(my $res = $nodelist->{$name})) {
 250             $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 251             $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
 252
 253             if ($res->{quorum_votes} == $param->{votes} &&
 254                 $res->{nodeid} == $param->{nodeid}) {
 255                 print "node $name already defined\n";
 256                 if ($param->{force}) {
 257                     exit (0);
 258                 } else {
 259                     exit (-1);
 260                 }
 261             } else {
 262                 die "can't add existing node\n";
 263             }
 264         } elsif (!$param->{nodeid}) {
 265             my $nodeid = 1;
 266
 267             while(1) {
 268                 my $found = 0;
 269                 foreach my $v (values %$nodelist) {
 270                     if ($v->{nodeid} eq $nodeid) {
 271                         $found = 1;
 272                         $nodeid++;
 273                         last;
 274                     }
 275                 }
 276                 last if !$found;
 277             };
 278
 279             $param->{nodeid} = $nodeid;
 280         }
 281
 282         $param->{votes} = 1 if !defined($param->{votes});
 283
 284         PVE::Cluster::gen_local_dirs($name);
 285
 286         eval {  PVE::Cluster::ssh_merge_keys(); };
 287         warn $@ if $@;
 288
 289         $nodelist->{$name} = { ring0_addr => $name, nodeid => $param->{nodeid} };
 290         $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
 291
 292         corosync_update_nodelist($conf, $nodelist);
 293
 294         exit (0);
 295     }});
 296
 297
 298 __PACKAGE__->register_method ({
 299     name => 'delnode',
 300     path => 'delnode',
 301     method => 'PUT',
 302     description => "Removes a node to the cluster configuration.",
 303     parameters => {
 304         additionalProperties => 0,
 305         properties => {
 306             node => PVE::JSONSchema::get_standard_option('pve-node'),
 307         },
 308     },
 309     returns => { type => 'null' },
 310
 311     code => sub {
 312         my ($param) = @_;
 313
 314         PVE::Cluster::check_cfs_quorum();
 315
 316         my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 317
 318         my $nodelist = corosync_nodelist($conf);
 319
 320         my $nd = delete $nodelist->{$param->{node}};
 321         die "no such node '$param->{node}'\n" if !$nd;
 322
 323         corosync_update_nodelist($conf, $nodelist);
 324
 325         return undef;
 326     }});
 327
 328 __PACKAGE__->register_method ({
 329     name => 'add',
 330     path => 'add',
 331     method => 'PUT',
 332     description => "Adds the current node to an existing cluster.",
 333     parameters => {
 334         additionalProperties => 0,
 335         properties => {
 336             hostname => {
 337                 type => 'string',
 338                 description => "Hostname (or IP) of an existing cluster member."
 339             },
 340             nodeid => {
 341                 type => 'integer',
 342                 description => "Node id for this node.",
 343                 minimum => 1,
 344                 optional => 1,
 345             },
 346             votes => {
 347                 type => 'integer',
 348                 description => "Number of votes for this node",
 349                 minimum => 0,
 350                 optional => 1,
 351             },
 352             force => {
 353                 type => 'boolean',
 354                 description => "Do not throw error if node already exists.",
 355                 optional => 1,
 356             },
 357         },
 358     },
 359     returns => { type => 'null' },
 360
 361     code => sub {
 362         my ($param) = @_;
 363
 364         PVE::Cluster::setup_sshd_config();
 365         PVE::Cluster::setup_rootsshconfig();
 366         PVE::Cluster::setup_ssh_keys();
 367
 368         my $host = $param->{hostname};
 369
 370         if (!$param->{force}) {
 371
 372             if (-f $authfile) {
 373                 die "authentication key already exists\n";
 374             }
 375
 376             if (-f $clusterconf)  {
 377                 die "cluster config '$clusterconf' already exists\n";
 378             }
 379
 380             my $vmlist = PVE::Cluster::get_vmlist();
 381             if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 382                 die "this host already contains virtual machines - please remove them first\n";
 383             }
 384
 385             if (system("corosync-quorumtool >/dev/null 2>&1") == 0) {
 386                 die "corosync is already running\n";
 387             }
 388         }
 389
 390         # make sure known_hosts is on local filesystem
 391         PVE::Cluster::ssh_unmerge_known_hosts();
 392
 393         my $cmd = "ssh-copy-id -i /root/.ssh/id_rsa 'root\@$host' >/dev/null 2>&1";
 394         system ($cmd) == 0 ||
 395             die "unable to copy ssh ID\n";
 396
 397         $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
 398                 'pvecm', 'addnode', $nodename, '--force', 1];
 399
 400         push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
 401
 402         push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
 403
 404         if (system (@$cmd) != 0) {
 405             my $cmdtxt = join (' ', @$cmd);
 406             die "unable to add node: command failed ($cmdtxt)\n";
 407         }
 408
 409         my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
 410         mkdir $tmpdir;
 411
 412         eval {
 413             print "copy corosync auth key\n";
 414             $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
 415                     "[$host]:$authfile $clusterconf", $tmpdir];
 416
 417             system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
 418
 419             mkdir "/etc/corosync";
 420             my $confbase = basename($clusterconf);
 421
 422             $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'";
 423             system($cmd) == 0 || die "can't copy cluster configuration\n";
 424
 425             my $keybase = basename($authfile);
 426             system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
 427                 die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
 428
 429             print "stopping pve-cluster service\n";
 430
 431             system("umount $basedir -f >/dev/null 2>&1");
 432             system("systemctl stop pve-cluster") == 0 ||
 433                 die "can't stop pve-cluster service\n";
 434
 435             backup_database();
 436
 437             unlink $dbfile;
 438
 439             system("systemctl start pve-cluster") == 0 ||
 440                 die "starting pve-cluster failed\n";
 441
 442             system("systemctl start corosync");
 443
 444             # wait for quorum
 445             my $printqmsg = 1;
 446             while (!PVE::Cluster::check_cfs_quorum(1)) {
 447                 if ($printqmsg) {
 448                     print "waiting for quorum...";
 449                     STDOUT->flush();
 450                     $printqmsg = 0;
 451                 }
 452                 sleep(1);
 453             }
 454             print "OK\n" if !$printqmsg;
 455
 456             # system("systemctl start clvm");
 457
 458             print "generating node certificates\n";
 459             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 460
 461             print "merge known_hosts file\n";
 462             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 463
 464             print "restart services\n";
 465             # restart pvedaemon (changed certs)
 466             system("systemctl restart pvedaemon");
 467             # restart pveproxy (changed certs)
 468             system("systemctl restart pveproxy");
 469
 470             print "successfully added node '$nodename' to cluster.\n";
 471         };
 472         my $err = $@;
 473
 474         rmtree $tmpdir;
 475
 476         die $err if $err;
 477
 478         return undef;
 479     }});
 480
 481 __PACKAGE__->register_method ({
 482     name => 'status',
 483     path => 'status',
 484     method => 'GET',
 485     description => "Displays the local view of the cluster status.",
 486     parameters => {
 487         additionalProperties => 0,
 488         properties => {},
 489     },
 490     returns => { type => 'null' },
 491
 492     code => sub {
 493         my ($param) = @_;
 494
 495         my $cmd = ['corosync-quorumtool', '-siH'];
 496
 497         exec (@$cmd);
 498
 499         exit (-1); # should not be reached
 500     }});
 501
 502 __PACKAGE__->register_method ({
 503     name => 'nodes',
 504     path => 'nodes',
 505     method => 'GET',
 506     description => "Displays the local view of the cluster nodes.",
 507     parameters => {
 508         additionalProperties => 0,
 509         properties => {},
 510     },
 511     returns => { type => 'null' },
 512
 513     code => sub {
 514         my ($param) = @_;
 515
 516         my $cmd = ['corosync-quorumtool', '-l'];
 517
 518         exec (@$cmd);
 519
 520         exit (-1); # should not be reached
 521     }});
 522
 523 __PACKAGE__->register_method ({
 524     name => 'expected',
 525     path => 'expected',
 526     method => 'PUT',
 527     description => "Tells corosync a new value of expected votes.",
 528     parameters => {
 529         additionalProperties => 0,
 530         properties => {
 531             expected => {
 532                 type => 'integer',
 533                 description => "Expected votes",
 534                 minimum => 1,
 535             },
 536         },
 537     },
 538     returns => { type => 'null' },
 539
 540     code => sub {
 541         my ($param) = @_;
 542
 543         my $cmd = ['corosync-quorumtool', '-e', $param->{expected}];
 544
 545         exec (@$cmd);
 546
 547         exit (-1); # should not be reached
 548
 549     }});
 550
 551 sub corosync_update_nodelist {
 552     my ($conf, $nodelist) = @_;
 553
 554     delete $conf->{digest};
 555
 556     my $version = PVE::Cluster::corosync_conf_version($conf);
 557     PVE::Cluster::corosync_conf_version($conf, undef, $version + 1);
 558
 559     my $children = [];
 560     foreach my $v (values %$nodelist) {
 561         next if !$v->{ring0_addr};
 562         my $kv = [];
 563         foreach my $k (keys %$v) {
 564             push @$kv, { key => $k, value => $v->{$k} };
 565         }
 566         my $ns = { section => 'node', children => $kv };
 567         push @$children, $ns;
 568     }
 569
 570     foreach my $main (@{$conf->{children}}) {
 571         next if !defined($main->{section});
 572         if ($main->{section} eq 'nodelist') {
 573             $main->{children} = $children;
 574             last;
 575         }
 576     }
 577
 578
 579     PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);
 580
 581     rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
 582         || die "activate  corosync.conf.new failed - $!\n";
 583 }
 584
 585 sub corosync_nodelist {
 586     my ($conf) = @_;
 587
 588     my $res = {};
 589
 590     my $nodelist = {};
 591
 592     foreach my $main (@{$conf->{children}}) {
 593         next if !defined($main->{section});
 594         if ($main->{section} eq 'nodelist') {
 595             foreach my $ne (@{$main->{children}}) {
 596                 next if !defined($ne->{section}) || ($ne->{section} ne 'node');
 597                 my $node = { quorum_votes => 1 };
 598                 foreach my $child (@{$ne->{children}}) {
 599                     next if !defined($child->{key});
 600                     $node->{$child->{key}} = $child->{value};
 601                     if ($child->{key} eq 'ring0_addr') {
 602                         $nodelist->{$child->{value}} = $node;
 603                     }
 604                 }
 605             }
 606         }
 607     }
 608
 609     return $nodelist;
 610 }
 611
 612 __PACKAGE__->register_method ({
 613     name => 'updatecerts',
 614     path => 'updatecerts',
 615     method => 'PUT',
 616     description => "Update node certificates (and generate all needed files/directories).",
 617     parameters => {
 618         additionalProperties => 0,
 619         properties => {
 620             force => {
 621                 description => "Force generation of new SSL certifate.",
 622                 type => 'boolean',
 623                 optional => 1,
 624             },
 625             silent => {
 626                 description => "Ignore errors (i.e. when cluster has no quorum).",
 627                 type => 'boolean',
 628                 optional => 1,
 629             },
 630         },
 631     },
 632     returns => { type => 'null' },
 633     code => sub {
 634         my ($param) = @_;
 635
 636         PVE::Cluster::setup_rootsshconfig();
 637
 638         PVE::Cluster::gen_pve_vzdump_symlink();
 639
 640         if (!PVE::Cluster::check_cfs_quorum(1)) {
 641             return undef if $param->{silent};
 642             die "no quorum - unable to update files\n";
 643         }
 644
 645         PVE::Cluster::setup_ssh_keys();
 646
 647         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
 648         PVE::Cluster::ssh_merge_keys();
 649         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
 650         PVE::Cluster::gen_pve_vzdump_files();
 651
 652         return undef;
 653     }});
 654
 655
 656 my $cmddef = {
 657     keygen => [ __PACKAGE__, 'keygen', ['filename']],
 658     create => [ __PACKAGE__, 'create', ['clustername']],
 659     add => [ __PACKAGE__, 'add', ['hostname']],
 660     addnode => [ __PACKAGE__, 'addnode', ['node']],
 661     delnode => [ __PACKAGE__, 'delnode', ['node']],
 662     status => [ __PACKAGE__, 'status' ],
 663     nodes => [ __PACKAGE__, 'nodes' ],
 664     expected => [ __PACKAGE__, 'expected', ['expected']],
 665     updatecerts => [ __PACKAGE__, 'updatecerts', []],
 666 };
 667
 668 my $cmd = shift;
 669
 670 if ($cmd && $cmd ne 'printmanpod' && $cmd ne 'verifyapi') {
 671     PVE::Cluster::check_cfs_is_mounted();
 672     PVE::Cluster::cfs_update();
 673 }
 674
 675 PVE::CLIHandler::handle_cmd($cmddef, "pvecm", $cmd, \@ARGV, undef, $0);
 676
 677 exit 0;
 678
 679 __END__
 680
 681 =head1 NAME
 682
 683 pvecm - Proxmox VE cluster manager toolkit
 684
 685 =head1 SYNOPSIS
 686
 687 =include synopsis
 688
 689 =head1 DESCRIPTION
 690
 691 pvecm is a program to manage the cluster configuration. It can be used
 692 to create a new cluster, join nodes to a cluster, leave the cluster,
 693 get status information and do various other cluster related tasks.
 694
 695 =include pve_copyright
 696
 697