data/PVE/pvecm

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5 use Getopt::Long;
   6 use Socket;
   7 use IO::File;
   8 use Net::IP;
   9 use File::Path;
  10 use File::Basename;
  11 use Data::Dumper; # fixme: remove
  12 use PVE::Tools;
  13 use PVE::Cluster;
  14 use PVE::INotify;
  15 use PVE::JSONSchema;
  16 use PVE::CLIHandler;
  17
  18 use base qw(PVE::CLIHandler);
  19
  20 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
  21 $ENV{LC_ALL} = 'C';
  22 $ENV{HOME} = '/root'; # for ssh-copy-id
  23
  24 my ($local_ip_address, $nodename);
  25
  26 if ($ARGV[0] ne 'printmanpod' && $ARGV[0] ne 'verifyapi') {
  27     die "please run as root\n" if $> != 0;
  28
  29     $nodename = PVE::INotify::nodename();
  30     # trigger check that we have resolvable name
  31     $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
  32 }
  33
  34 my $basedir = "/etc/pve";
  35 my $clusterconf = "$basedir/corosync.conf";
  36 my $libdir = "/var/lib/pve-cluster";
  37 my $backupdir = "/var/lib/pve-cluster/backup";
  38 my $dbfile = "$libdir/config.db";
  39 my $authfile = "/etc/corosync/authkey";
  40
  41
  42 sub backup_database {
  43
  44     print "backup old database\n";
  45
  46     mkdir $backupdir;
  47
  48     my $ctime = time();
  49     my $cmd = "echo '.dump' |";
  50     $cmd .= "sqlite3 '$dbfile' |";
  51     $cmd .= "gzip - >'${backupdir}/config-${ctime}.sql.gz'";
  52
  53     system($cmd) == 0 ||
  54         die "can't backup old database: $!\n";
  55
  56     # purge older backup
  57     my $maxfiles = 10;
  58
  59     my @bklist = ();
  60     foreach my $fn (<$backupdir/config-*.sql.gz>) {
  61         if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
  62             push @bklist, [$fn, $1];
  63         }
  64     }
  65
  66     @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
  67
  68     while (scalar (@bklist) >= $maxfiles) {
  69         my $d = pop @bklist;
  70         print "delete old backup '$d->[0]'\n";
  71         unlink $d->[0];
  72     }
  73 }
  74
  75 __PACKAGE__->register_method ({
  76     name => 'keygen',
  77     path => 'keygen',
  78     method => 'PUT',
  79     description => "Generate new cryptographic key for corosync.",
  80     parameters => {
  81         additionalProperties => 0,
  82         properties => {
  83             filename => {
  84                 type => 'string',
  85                 description => "Output file name"
  86             }
  87         },
  88     },
  89     returns => { type => 'null' },
  90
  91     code => sub {
  92         my ($param) = @_;
  93
  94         my $filename = $param->{filename};
  95
  96         # test EUID
  97         $> == 0 || die "Error: Authorization key must be generated as root user.\n";
  98         my $dirname = dirname($filename);
  99         my $basename = basename($filename);
 100
 101         die "key file '$filename' already exists\n" if -e $filename;
 102
 103         File::Path::make_path($dirname) if $dirname;
 104
 105         my $cmd = ['corosync-keygen', '-l', '-k', $filename];
 106         PVE::Tools::run_command($cmd);
 107
 108         return undef;
 109     }});
 110
 111 __PACKAGE__->register_method ({
 112     name => 'create',
 113     path => 'create',
 114     method => 'PUT',
 115     description => "Generate new cluster configuration.",
 116     parameters => {
 117         additionalProperties => 0,
 118         properties => {
 119             clustername => {
 120                 description => "The name of the cluster.",
 121                 type => 'string', format => 'pve-node',
 122                 maxLength => 15,
 123             },
 124             nodeid => {
 125                 type => 'integer',
 126                 description => "Node id for this node.",
 127                 minimum => 1,
 128                 optional => 1,
 129             },
 130             votes => {
 131                 type => 'integer',
 132                 description => "Number of votes for this node.",
 133                 minimum => 1,
 134                 optional => 1,
 135             },
 136         },
 137     },
 138     returns => { type => 'null' },
 139
 140     code => sub {
 141         my ($param) = @_;
 142
 143         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
 144
 145         PVE::Cluster::setup_rootsshconfig();
 146         PVE::Cluster::setup_ssh_keys();
 147
 148         -f $authfile || __PACKAGE__->keygen({filename => $authfile});
 149
 150         -f $authfile || die "no authentication key available\n";
 151
 152         my $clustername = $param->{clustername};
 153
 154         $param->{nodeid} = 1 if !$param->{nodeid};
 155
 156         $param->{votes} = 1 if !defined($param->{votes});
 157
 158         # No, corosync cannot deduce this on its own
 159         my $ipversion = Net::IP::ip_is_ipv6($local_ip_address) ? 'ipv6' : 'ipv4';
 160
 161         my $config = <<_EOD;
 162 totem {
 163   version: 2
 164   secauth: on
 165   cluster_name: $clustername
 166   config_version: 1
 167   ip_version: $ipversion
 168   interface {
 169     ringnumber: 0
 170     bindnetaddr: $local_ip_address
 171   }
 172 }
 173
 174 nodelist {
 175   node {
 176     ring0_addr: $nodename
 177     nodeid: $param->{nodeid}
 178     quorum_votes: $param->{votes}
 179   }
 180 }
 181
 182 quorum {
 183   provider: corosync_votequorum
 184 }
 185
 186 logging {
 187   to_syslog: yes
 188   debug: off
 189 }
 190 _EOD
 191 ;
 192         PVE::Tools::file_set_contents($clusterconf, $config);
 193
 194         PVE::Cluster::ssh_merge_keys();
 195
 196         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 197
 198         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 199
 200         PVE::Tools::run_command('service pve-cluster restart'); # restart
 201
 202         PVE::Tools::run_command('service corosync restart'); # restart
 203
 204         return undef;
 205 }});
 206
 207 __PACKAGE__->register_method ({
 208     name => 'addnode',
 209     path => 'addnode',
 210     method => 'PUT',
 211     description => "Adds a node to the cluster configuration.",
 212     parameters => {
 213         additionalProperties => 0,
 214         properties => {
 215             node => PVE::JSONSchema::get_standard_option('pve-node'),
 216             nodeid => {
 217                 type => 'integer',
 218                 description => "Node id for this node.",
 219                 minimum => 1,
 220                 optional => 1,
 221             },
 222             votes => {
 223                 type => 'integer',
 224                 description => "Number of votes for this node",
 225                 minimum => 0,
 226                 optional => 1,
 227             },
 228             force => {
 229                 type => 'boolean',
 230                 description => "Do not throw error if node already exists.",
 231                 optional => 1,
 232             },
 233         },
 234     },
 235     returns => { type => 'null' },
 236
 237     code => sub {
 238         my ($param) = @_;
 239
 240         PVE::Cluster::check_cfs_quorum();
 241
 242         my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 243
 244         my $nodelist = corosync_nodelist($conf);
 245
 246         my $name = $param->{node};
 247
 248         if (defined(my $res = $nodelist->{$name})) {
 249             $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 250             $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
 251
 252             if ($res->{quorum_votes} == $param->{votes} &&
 253                 $res->{nodeid} == $param->{nodeid}) {
 254                 print "node $name already defined\n";
 255                 if ($param->{force}) {
 256                     exit (0);
 257                 } else {
 258                     exit (-1);
 259                 }
 260             } else {
 261                 die "can't add existing node\n";
 262             }
 263         } elsif (!$param->{nodeid}) {
 264             my $nodeid = 1;
 265
 266             while(1) {
 267                 my $found = 0;
 268                 foreach my $v (values %$nodelist) {
 269                     if ($v->{nodeid} eq $nodeid) {
 270                         $found = 1;
 271                         $nodeid++;
 272                         last;
 273                     }
 274                 }
 275                 last if !$found;
 276             };
 277
 278             $param->{nodeid} = $nodeid;
 279         }
 280
 281         $param->{votes} = 1 if !defined($param->{votes});
 282
 283         PVE::Cluster::gen_local_dirs($name);
 284
 285         eval {  PVE::Cluster::ssh_merge_keys(); };
 286         warn $@ if $@;
 287
 288         $nodelist->{$name} = { ring0_addr => $name, nodeid => $param->{nodeid} };
 289         $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
 290
 291         corosync_update_nodelist($conf, $nodelist);
 292
 293         exit (0);
 294     }});
 295
 296
 297 __PACKAGE__->register_method ({
 298     name => 'delnode',
 299     path => 'delnode',
 300     method => 'PUT',
 301     description => "Removes a node to the cluster configuration.",
 302     parameters => {
 303         additionalProperties => 0,
 304         properties => {
 305             node => PVE::JSONSchema::get_standard_option('pve-node'),
 306         },
 307     },
 308     returns => { type => 'null' },
 309
 310     code => sub {
 311         my ($param) = @_;
 312
 313         PVE::Cluster::check_cfs_quorum();
 314
 315         my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 316
 317         my $nodelist = corosync_nodelist($conf);
 318
 319         my $nd = delete $nodelist->{$param->{node}};
 320         die "no such node '$param->{node}'\n" if !$nd;
 321
 322         corosync_update_nodelist($conf, $nodelist);
 323
 324         return undef;
 325     }});
 326
 327 __PACKAGE__->register_method ({
 328     name => 'add',
 329     path => 'add',
 330     method => 'PUT',
 331     description => "Adds the current node to an existing cluster.",
 332     parameters => {
 333         additionalProperties => 0,
 334         properties => {
 335             hostname => {
 336                 type => 'string',
 337                 description => "Hostname (or IP) of an existing cluster member."
 338             },
 339             nodeid => {
 340                 type => 'integer',
 341                 description => "Node id for this node.",
 342                 minimum => 1,
 343                 optional => 1,
 344             },
 345             votes => {
 346                 type => 'integer',
 347                 description => "Number of votes for this node",
 348                 minimum => 0,
 349                 optional => 1,
 350             },
 351             force => {
 352                 type => 'boolean',
 353                 description => "Do not throw error if node already exists.",
 354                 optional => 1,
 355             },
 356         },
 357     },
 358     returns => { type => 'null' },
 359
 360     code => sub {
 361         my ($param) = @_;
 362
 363         PVE::Cluster::setup_rootsshconfig();
 364         PVE::Cluster::setup_ssh_keys();
 365
 366         my $host = $param->{hostname};
 367
 368         if (!$param->{force}) {
 369
 370             if (-f $authfile) {
 371                 die "authentication key already exists\n";
 372             }
 373
 374             if (-f $clusterconf)  {
 375                 die "cluster config '$clusterconf' already exists\n";
 376             }
 377
 378             my $vmlist = PVE::Cluster::get_vmlist();
 379             if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 380                 die "this host already contains virtual machines - please remove the first\n";
 381             }
 382
 383             if (system("corosync-quorumtool >/dev/null 2>&1") == 0) {
 384                 die "corosync is already running\n";
 385             }
 386         }
 387
 388         # make sure known_hosts is on local filesystem
 389         PVE::Cluster::ssh_unmerge_known_hosts();
 390
 391         my $cmd = "ssh-copy-id -i /root/.ssh/id_rsa 'root\@$host' >/dev/null 2>&1";
 392         system ($cmd) == 0 ||
 393             die "unable to copy ssh ID\n";
 394
 395         $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
 396                 'pvecm', 'addnode', $nodename, '--force', 1];
 397
 398         push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
 399
 400         push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
 401
 402         if (system (@$cmd) != 0) {
 403             my $cmdtxt = join (' ', @$cmd);
 404             die "unable to add node: command failed ($cmdtxt)\n";
 405         }
 406
 407         my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
 408         mkdir $tmpdir;
 409
 410         eval {
 411             print "copy corosync auth key\n";
 412             $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
 413                     "$host:$authfile $clusterconf", $tmpdir];
 414
 415             system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
 416
 417             mkdir "/etc/corosync";
 418             my $confbase = basename($clusterconf);
 419
 420             $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'";
 421             system($cmd) == 0 || die "can't copy cluster configuration\n";
 422
 423             my $keybase = basename($authfile);
 424             system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
 425                 die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
 426
 427             print "stopping pve-cluster service\n";
 428
 429             system("umount $basedir -f >/dev/null 2>&1");
 430             system("systemctl stop pve-cluster") == 0 ||
 431                 die "can't stop pve-cluster service\n";
 432
 433             backup_database();
 434
 435             unlink $dbfile;
 436
 437             system("systemctl start pve-cluster") == 0 ||
 438                 die "starting pve-cluster failed\n";
 439
 440             system("systemctl start corosync");
 441
 442             # wait for quorum
 443             my $printqmsg = 1;
 444             while (!PVE::Cluster::check_cfs_quorum(1)) {
 445                 if ($printqmsg) {
 446                     print "waiting for quorum...";
 447                     STDOUT->flush();
 448                     $printqmsg = 0;
 449                 }
 450                 sleep(1);
 451             }
 452             print "OK\n" if !$printqmsg;
 453
 454             # system("systemctl start clvm");
 455
 456             print "generating node certificates\n";
 457             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 458
 459             print "merge known_hosts file\n";
 460             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 461
 462             print "restart services\n";
 463             # restart pvedaemon (changed certs)
 464             system("systemctl restart pvedaemon");
 465             # restart pveproxy (changed certs)
 466             system("systemctl restart pveproxy");
 467
 468             print "successfully added node '$nodename' to cluster.\n";
 469         };
 470         my $err = $@;
 471
 472         rmtree $tmpdir;
 473
 474         die $err if $err;
 475
 476         return undef;
 477     }});
 478
 479 __PACKAGE__->register_method ({
 480     name => 'status',
 481     path => 'status',
 482     method => 'GET',
 483     description => "Displays the local view of the cluster status.",
 484     parameters => {
 485         additionalProperties => 0,
 486         properties => {},
 487     },
 488     returns => { type => 'null' },
 489
 490     code => sub {
 491         my ($param) = @_;
 492
 493         my $cmd = ['corosync-quorumtool', '-siH'];
 494
 495         exec (@$cmd);
 496
 497         exit (-1); # should not be reached
 498     }});
 499
 500 __PACKAGE__->register_method ({
 501     name => 'nodes',
 502     path => 'nodes',
 503     method => 'GET',
 504     description => "Displays the local view of the cluster nodes.",
 505     parameters => {
 506         additionalProperties => 0,
 507         properties => {},
 508     },
 509     returns => { type => 'null' },
 510
 511     code => sub {
 512         my ($param) = @_;
 513
 514         my $cmd = ['corosync-quorumtool', '-l'];
 515
 516         exec (@$cmd);
 517
 518         exit (-1); # should not be reached
 519     }});
 520
 521 __PACKAGE__->register_method ({
 522     name => 'expected',
 523     path => 'expected',
 524     method => 'PUT',
 525     description => "Tells corosync a new value of expected votes.",
 526     parameters => {
 527         additionalProperties => 0,
 528         properties => {
 529             expected => {
 530                 type => 'integer',
 531                 description => "Expected votes",
 532                 minimum => 1,
 533             },
 534         },
 535     },
 536     returns => { type => 'null' },
 537
 538     code => sub {
 539         my ($param) = @_;
 540
 541         my $cmd = ['corosync-quorumtool', '-e', $param->{expected}];
 542
 543         exec (@$cmd);
 544
 545         exit (-1); # should not be reached
 546
 547     }});
 548
 549 sub corosync_update_nodelist {
 550     my ($conf, $nodelist) = @_;
 551
 552     delete $conf->{digest};
 553
 554     my $version = PVE::Cluster::corosync_conf_version($conf);
 555     PVE::Cluster::corosync_conf_version($conf, undef, $version + 1);
 556
 557     my $children = [];
 558     foreach my $v (values %$nodelist) {
 559         next if !$v->{ring0_addr};
 560         my $kv = [];
 561         foreach my $k (keys %$v) {
 562             push @$kv, { key => $k, value => $v->{$k} };
 563         }
 564         my $ns = { section => 'node', children => $kv };
 565         push @$children, $ns;
 566     }
 567
 568     foreach my $main (@{$conf->{children}}) {
 569         next if !defined($main->{section});
 570         if ($main->{section} eq 'nodelist') {
 571             $main->{children} = $children;
 572             last;
 573         }
 574     }
 575
 576
 577     PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);
 578
 579     rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
 580         || die "activate  corosync.conf.new failed - $!\n";
 581 }
 582
 583 sub corosync_nodelist {
 584     my ($conf) = @_;
 585
 586     my $res = {};
 587
 588     my $nodelist = {};
 589
 590     foreach my $main (@{$conf->{children}}) {
 591         next if !defined($main->{section});
 592         if ($main->{section} eq 'nodelist') {
 593             foreach my $ne (@{$main->{children}}) {
 594                 next if !defined($ne->{section}) || ($ne->{section} ne 'node');
 595                 my $node = { quorum_votes => 1 };
 596                 foreach my $child (@{$ne->{children}}) {
 597                     next if !defined($child->{key});
 598                     $node->{$child->{key}} = $child->{value};
 599                     if ($child->{key} eq 'ring0_addr') {
 600                         $nodelist->{$child->{value}} = $node;
 601                     }
 602                 }
 603             }
 604         }
 605     }
 606
 607     return $nodelist;
 608 }
 609
 610 __PACKAGE__->register_method ({
 611     name => 'updatecerts',
 612     path => 'updatecerts',
 613     method => 'PUT',
 614     description => "Update node certificates (and generate all needed files/directories).",
 615     parameters => {
 616         additionalProperties => 0,
 617         properties => {
 618             force => {
 619                 description => "Force generation of new SSL certifate.",
 620                 type => 'boolean',
 621                 optional => 1,
 622             },
 623             silent => {
 624                 description => "Ignore errors (i.e. when cluster has no quorum).",
 625                 type => 'boolean',
 626                 optional => 1,
 627             },
 628         },
 629     },
 630     returns => { type => 'null' },
 631     code => sub {
 632         my ($param) = @_;
 633
 634         PVE::Cluster::setup_rootsshconfig();
 635
 636         PVE::Cluster::gen_pve_vzdump_symlink();
 637
 638         if (!PVE::Cluster::check_cfs_quorum(1)) {
 639             return undef if $param->{silent};
 640             die "no quorum - unable to update files\n";
 641         }
 642
 643         PVE::Cluster::setup_ssh_keys();
 644
 645         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
 646         PVE::Cluster::ssh_merge_keys();
 647         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
 648         PVE::Cluster::gen_pve_vzdump_files();
 649
 650         return undef;
 651     }});
 652
 653
 654 my $cmddef = {
 655     keygen => [ __PACKAGE__, 'keygen', ['filename']],
 656     create => [ __PACKAGE__, 'create', ['clustername']],
 657     add => [ __PACKAGE__, 'add', ['hostname']],
 658     addnode => [ __PACKAGE__, 'addnode', ['node']],
 659     delnode => [ __PACKAGE__, 'delnode', ['node']],
 660     status => [ __PACKAGE__, 'status' ],
 661     nodes => [ __PACKAGE__, 'nodes' ],
 662     expected => [ __PACKAGE__, 'expected', ['expected']],
 663     updatecerts => [ __PACKAGE__, 'updatecerts', []],
 664 };
 665
 666 my $cmd = shift;
 667
 668 if ($cmd && $cmd ne 'printmanpod' && $cmd ne 'verifyapi') {
 669     PVE::Cluster::check_cfs_is_mounted();
 670     PVE::Cluster::cfs_update();
 671 }
 672
 673 PVE::CLIHandler::handle_cmd($cmddef, "pvecm", $cmd, \@ARGV, undef, $0);
 674
 675 exit 0;
 676
 677 __END__
 678
 679 =head1 NAME
 680
 681 pvecm - Proxmox VE cluster manager toolkit
 682
 683 =head1 SYNOPSIS
 684
 685 =include synopsis
 686
 687 =head1 DESCRIPTION
 688
 689 pvecm is a program to manage the cluster configuration. It can be used
 690 to create a new cluster, join nodes to a cluster, leave the cluster,
 691 get status information and do various other cluster related tasks.
 692
 693 =include pve_copyright
 694
 695