data/PVE/pvecm

   1 #!/usr/bin/perl -w
   2 use strict;
   3 use Getopt::Long;
   4 use Socket;
   5 use IO::File;
   6 use File::Path;
   7 use File::Basename;
   8 use Data::Dumper; # fixme: remove
   9 use PVE::Tools;
  10 use PVE::Cluster;
  11 use PVE::INotify;
  12 use PVE::JSONSchema;
  13 use PVE::CLIHandler;
  14
  15 use base qw(PVE::CLIHandler);
  16
  17 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
  18
  19 die "please run as root\n" if $> != 0;
  20
  21 my $nodename = PVE::INotify::nodename();
  22 # trigger check that we have resolvable name
  23 my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
  24
  25 my $basedir = "/etc/pve";
  26 my $clusterconf = "$basedir/cluster.conf";
  27 my $libdir = "/var/lib/pve-cluster";
  28 my $backupdir = "/var/lib/pve-cluster/backup";
  29 my $dbfile = "$libdir/config.db";
  30 my $authfile = "$libdir/corosync.authkey";
  31
  32
  33 sub backup_database {
  34
  35     print "backup old database\n";
  36
  37     mkdir $backupdir;
  38
  39     my $ctime = time();
  40     my $cmd = "echo '.dump' |";
  41     $cmd .= "sqlite3 '$dbfile' |";
  42     $cmd .= "gzip - >'${backupdir}/config-${ctime}.sql.gz'";
  43
  44     system($cmd) == 0 ||
  45         die "can't backup old database: $!\n";
  46
  47     # purge older backup
  48     my $maxfiles = 10;
  49
  50     my @bklist = ();
  51     foreach my $fn (<$backupdir/config-*.sql.gz>) {
  52         if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
  53             push @bklist, [$fn, $1];
  54         }
  55     }
  56
  57     @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
  58
  59     while (scalar (@bklist) >= $maxfiles) {
  60         my $d = pop @bklist;
  61         print "delete old backup '$d->[0]'\n";
  62         unlink $d->[0];
  63     }
  64 }
  65
  66 __PACKAGE__->register_method ({
  67     name => 'keygen',
  68     path => 'keygen',
  69     method => 'PUT',
  70     description => "Generate new cryptographic key for corosync.",
  71     parameters => {
  72         additionalProperties => 0,
  73         properties => {
  74             filename => {
  75                 type => 'string',
  76                 description => "Output file name"
  77             }
  78         },
  79     },
  80     returns => { type => 'null' },
  81
  82     code => sub {
  83         my ($param) = @_;
  84
  85         my $filename = $param->{filename};
  86
  87         # test EUID
  88         $> == 0 || die "Error: Authorization key must be generated as root user.\n";
  89         my $dirname = dirname($filename);
  90         my $basename = basename($filename);
  91
  92         File::Path::make_path($dirname) if $dirname;
  93
  94         my $fh = IO::File->new ("/dev/urandom", 'r') ||
  95             die "can't open /dev/urandom - $!\n";
  96
  97         my $keysize = 128;
  98         my $key = '';
  99
 100         my $bytes = 0;
 101         while ($bytes < $keysize) {
 102             my $rb = sysread($fh, $key, $keysize - $bytes, $bytes);
 103             ($rb > 0) || die "Could not read /dev/urandom - $!\n";
 104             $bytes += $rb;
 105         }
 106
 107         close($fh);
 108
 109         my $tmpfn = "$filename.tmp.$$";
 110         $fh = IO::File->new ($tmpfn, O_CREAT|O_WRONLY, 0400);
 111         die "can't open temporary file '$tmpfn' - $!\n" if !$fh;
 112
 113         eval {
 114             my $wb;
 115             (($wb = syswrite($fh, $key)) == $keysize) ||
 116                 die "writing key failed - short write $wb\n;"
 117         };
 118         my $err = $@;
 119
 120         $fh->close();
 121
 122         if ($err) {
 123             unlink $tmpfn;
 124             die $err;
 125         }
 126
 127         if (!rename($tmpfn, $filename)) {
 128             my $err = $!;
 129             unlink $tmpfn;
 130             die "rename $tmpfn to $filename failed - $err\n";
 131         }
 132
 133         return undef;
 134     }});
 135
 136 __PACKAGE__->register_method ({
 137     name => 'create',
 138     path => 'create',
 139     method => 'PUT',
 140     description => "Generate new cluster configuration.",
 141     parameters => {
 142         additionalProperties => 0,
 143         properties => {
 144             clustername => {
 145                 description => "The name of the cluster.",
 146                 type => 'string', format => 'pve-node',
 147                 maxLength => 15,
 148             },
 149             nodeid => {
 150                 type => 'integer',
 151                 description => "Node id for this node.",
 152                 minimum => 1,
 153                 optional => 1,
 154             },
 155             votes => {
 156                 type => 'integer',
 157                 description => "Number of votes for this node",
 158                 minimum => 1,
 159                 optional => 1,
 160             },
 161         },
 162     },
 163     returns => { type => 'null' },
 164
 165     code => sub {
 166         my ($param) = @_;
 167
 168         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
 169
 170         PVE::Cluster::setup_ssh_keys();
 171
 172         -f $authfile || __PACKAGE__->keygen({filename => $authfile});
 173
 174         -f $authfile || die "no authentication key available\n";
 175
 176         my $clustername = $param->{clustername};
 177
 178         $param->{nodeid} = 1 if !$param->{nodeid};
 179
 180         $param->{votes} = 1 if !defined($param->{votes});
 181
 182         my $config = <<_EOD;
 183 <?xml version="1.0"?>
 184 <cluster name="$clustername" config_version="1">
 185
 186   <cman keyfile="$authfile">
 187   </cman>
 188
 189   <clusternodes>
 190   <clusternode name="${nodename}" votes="$param->{votes}" nodeid="$param->{nodeid}"/>
 191   </clusternodes>
 192
 193 </cluster>
 194 _EOD
 195 ;
 196         PVE::Tools::file_set_contents($clusterconf, $config);
 197
 198         PVE::Cluster::ssh_merge_keys();
 199
 200         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 201
 202         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 203
 204         PVE::Tools::run_command('/etc/init.d/pve-cluster restart'); # restart
 205
 206         # that cman init script returns strange values - simply ignore for now
 207         system('/etc/init.d/cman start');
 208
 209         # also start clvm
 210         system('/etc/init.d/clvm start');
 211
 212         return undef;
 213 }});
 214
 215 __PACKAGE__->register_method ({
 216     name => 'addnode',
 217     path => 'addnode',
 218     method => 'PUT',
 219     description => "Adds a node to the cluster configuration.",
 220     parameters => {
 221         additionalProperties => 0,
 222         properties => {
 223             node => PVE::JSONSchema::get_standard_option('pve-node'),
 224             nodeid => {
 225                 type => 'integer',
 226                 description => "Node id for this node.",
 227                 minimum => 1,
 228                 optional => 1,
 229             },
 230             votes => {
 231                 type => 'integer',
 232                 description => "Number of votes for this node",
 233                 minimum => 0,
 234                 optional => 1,
 235             },
 236             force => {
 237                 type => 'boolean',
 238                 description => "Do not throw error if node already exists.",
 239                 optional => 1,
 240             },
 241         },
 242     },
 243     returns => { type => 'null' },
 244
 245     code => sub {
 246         my ($param) = @_;
 247
 248         PVE::Cluster::check_cfs_quorum();
 249
 250         my $lst = lsnode();
 251
 252         my $name = $param->{node};
 253
 254         if (defined(my $res = $lst->{$name})) {
 255             $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 256             $param->{votes} = $res->{votes} if !defined($param->{votes});
 257
 258             if ($res->{votes} == $param->{votes} &&
 259                 $res->{nodeid} == $param->{nodeid}) {
 260                 print "node $name already defined\n";
 261                 if ($param->{force}) {
 262                     exit (0);
 263                 } else {
 264                     exit (-1);
 265                 }
 266             }
 267         } elsif (!$param->{nodeid}) {
 268             my $nodeid = 1;
 269
 270             while(1) {
 271                 my $found = 0;
 272                 foreach my $v (values %$lst) {
 273                     if ($v->{nodeid} eq $nodeid) {
 274                         $found = 1;
 275                         $nodeid++;
 276                         last;
 277                     }
 278                 }
 279                 last if !$found;
 280             };
 281
 282             $param->{nodeid} = $nodeid;
 283         }
 284
 285         $param->{votes} = 1 if !defined($param->{votes});
 286
 287         PVE::Cluster::gen_local_dirs($name);
 288
 289         eval {  PVE::Cluster::ssh_merge_keys(); };
 290         warn $@ if $@;
 291
 292         my $cmd = ['ccs_tool', 'addnode', '-c', $clusterconf];
 293
 294         # NOTE: cman does not like votes="0"
 295         if ($param->{votes}) {
 296             push @$cmd, '-v', $param->{votes};
 297         }
 298
 299         push @$cmd, '-n', $param->{nodeid}, $name;
 300
 301         system(@$cmd) == 0 || exit(-1);
 302
 303         exit (0);
 304     }});
 305
 306
 307 __PACKAGE__->register_method ({
 308     name => 'delnode',
 309     path => 'delnode',
 310     method => 'PUT',
 311     description => "Removes a node to the cluster configuration.",
 312     parameters => {
 313         additionalProperties => 0,
 314         properties => {
 315             node => PVE::JSONSchema::get_standard_option('pve-node'),
 316         },
 317     },
 318     returns => { type => 'null' },
 319
 320     code => sub {
 321         my ($param) = @_;
 322
 323         die "cluster not ready - no quorum?\n"
 324             if !PVE::Cluster::check_cfs_quorum();
 325
 326         my $cmd = ['ccs_tool', 'delnode', '-c', $clusterconf, $param->{node}];
 327
 328         exec (@$cmd);
 329
 330         exit (-1); # should not be reached
 331     }});
 332
 333 __PACKAGE__->register_method ({
 334     name => 'add',
 335     path => 'add',
 336     method => 'PUT',
 337     description => "Adds the current node to an existing cluster.",
 338     parameters => {
 339         additionalProperties => 0,
 340         properties => {
 341             hostname => {
 342                 type => 'string',
 343                 description => "Hostname (or IP) of an existing cluster member."
 344             },
 345             nodeid => {
 346                 type => 'integer',
 347                 description => "Node id for this node.",
 348                 minimum => 1,
 349                 optional => 1,
 350             },
 351             votes => {
 352                 type => 'integer',
 353                 description => "Number of votes for this node",
 354                 minimum => 0,
 355                 optional => 1,
 356             },
 357             force => {
 358                 type => 'boolean',
 359                 description => "Do not throw error if node already exists.",
 360                 optional => 1,
 361             },
 362         },
 363     },
 364     returns => { type => 'null' },
 365
 366     code => sub {
 367         my ($param) = @_;
 368
 369         PVE::Cluster::setup_ssh_keys();
 370
 371         my $host = $param->{hostname};
 372
 373         if (!$param->{force}) {
 374
 375             if (-f $authfile) {
 376                 die "authentication key already exists\n";
 377             }
 378
 379             if (-f $clusterconf)  {
 380                 die "cluster config '$clusterconf' already exists\n";
 381             }
 382
 383             my $vmlist = PVE::Cluster::get_vmlist();
 384             if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 385                 die "this host already contains virtual machines - please remove the first\n";
 386             }
 387
 388             if (system("cman_tool status >/dev/null 2>&1") == 0) {
 389                 die "cman is already running\n";
 390             }
 391         }
 392
 393         # make sure known_hosts is on local filesystem
 394         PVE::Cluster::ssh_unmerge_known_hosts();
 395
 396         my $cmd = "ssh-copy-id 'root\@$host' >/dev/null 2>&1";
 397         system ($cmd) == 0 ||
 398             die "unable to copy ssh ID\n";
 399
 400         $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
 401                 'pvecm', 'addnode', $nodename, '--force', 1];
 402
 403         push @$cmd, '-n', $param->{nodeid} if $param->{nodeid};
 404
 405         push @$cmd, '-v', $param->{votes} if defined($param->{votes});
 406
 407         if (system (@$cmd) != 0) {
 408             my $cmdtxt = join (' ', @$cmd);
 409             die "unable to add node: command failed ($cmdtxt)\n";
 410         }
 411
 412         my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
 413         mkdir $tmpdir;
 414
 415         eval {
 416             print "copy corosync auth key\n";
 417             $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
 418                     "$host:$authfile $clusterconf", $tmpdir];
 419
 420             system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
 421
 422             mkdir "/etc/cluster";
 423             my $confbase = basename($clusterconf);
 424
 425             $cmd = "cp '$tmpdir/$confbase' '/etc/cluster/$confbase'";
 426             system($cmd) == 0 || die "can't copy cluster configuration\n";
 427
 428             my $keybase = basename($authfile);
 429             system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
 430                 die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
 431
 432             print "stopping pve-cluster service\n";
 433
 434             system("umount $basedir -f >/dev/null 2>&1");
 435             system("/etc/init.d/pve-cluster stop") == 0 ||
 436                 die "can't stop pve-cluster service\n";
 437
 438             backup_database();
 439
 440             unlink $dbfile;
 441
 442             system("/etc/init.d/pve-cluster start") == 0 ||
 443                 die "starting pve-cluster failed\n";
 444
 445             system("/etc/init.d/cman start");
 446
 447             # wait for quorum
 448             my $printqmsg = 1;
 449             while (!PVE::Cluster::check_cfs_quorum()) {
 450                 if ($printqmsg) {
 451                     print "waiting for quorum...";
 452                     STDOUT->flush();
 453                     $printqmsg = 0;
 454                 }
 455                 sleep(1);
 456             }
 457             print "OK\n" if !$printqmsg;
 458
 459             system("/etc/init.d/clvm start");
 460
 461             print "generating node certificates\n";
 462             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 463
 464             print "merge known_hosts file\n";
 465             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 466
 467             print "restart services\n";
 468             # restart apache (changed certs)
 469             system("/etc/init.d/apache2 restart");
 470
 471             print "successfully added node '$nodename' to cluster.\n";
 472         };
 473         my $err = $@;
 474
 475         rmtree $tmpdir;
 476
 477         die $err if $err;
 478
 479         return undef;
 480     }});
 481
 482 __PACKAGE__->register_method ({
 483     name => 'status',
 484     path => 'status',
 485     method => 'GET',
 486     description => "Displays the local view of the cluster status.",
 487     parameters => {
 488         additionalProperties => 0,
 489         properties => {},
 490     },
 491     returns => { type => 'null' },
 492
 493     code => sub {
 494         my ($param) = @_;
 495
 496         my $cmd = ['cman_tool', 'status'];
 497
 498         exec (@$cmd);
 499
 500         exit (-1); # should not be reached
 501     }});
 502
 503 __PACKAGE__->register_method ({
 504     name => 'nodes',
 505     path => 'nodes',
 506     method => 'GET',
 507     description => "Displays the local view of the cluster nodes.",
 508     parameters => {
 509         additionalProperties => 0,
 510         properties => {},
 511     },
 512     returns => { type => 'null' },
 513
 514     code => sub {
 515         my ($param) = @_;
 516
 517         my $cmd = ['cman_tool', 'nodes'];
 518
 519         exec (@$cmd);
 520
 521         exit (-1); # should not be reached
 522     }});
 523
 524 __PACKAGE__->register_method ({
 525     name => 'expected',
 526     path => 'expected',
 527     method => 'PUT',
 528     description => "Tells CMAN a new value of expected votes.",
 529     parameters => {
 530         additionalProperties => 0,
 531         properties => {
 532             expected => {
 533                 type => 'integer',
 534                 description => "Expected votes",
 535                 minimum => 1,
 536             },
 537         },
 538     },
 539     returns => { type => 'null' },
 540
 541     code => sub {
 542         my ($param) = @_;
 543
 544         my $cmd = ['cman_tool', 'expected', '-e', $param->{expected}];
 545
 546         exec (@$cmd);
 547
 548         exit (-1); # should not be reached
 549
 550     }});
 551
 552 sub lsnode {
 553
 554     my $res = {};
 555
 556     my $cmd = ['ccs_tool',  'lsnode', '-c', $clusterconf];
 557
 558     my $scan = 0;
 559     my $parser = sub {
 560         my $line = shift;
 561
 562         if ($line =~ m/^Nodename\s+Votes\s+Nodeid\s/i) {
 563             $scan = 1;
 564             return;
 565         }
 566         return if !$scan;
 567         if ($line =~ m/^(\S+)\s+(\d+)\s+(\d+)\s/) {
 568             $res->{$1} = {
 569                 name => $1,
 570                 votes => $2,
 571                 nodeid => $3 };
 572         }
 573     };
 574
 575     PVE::Tools::run_command($cmd, outfunc => $parser);
 576     return $res;
 577 }
 578
 579 __PACKAGE__->register_method ({
 580     name => 'updatecerts',
 581     path => 'updatecerts',
 582     method => 'PUT',
 583     description => "Update node certificates (and generate all needed files/directories).",
 584     parameters => {
 585         additionalProperties => 0,
 586         properties => {
 587             force => {
 588                 description => "Force generation of new SSL certifate.",
 589                 type => 'boolean',
 590                 optional => 1,
 591             },
 592             silent => {
 593                 description => "Ignore errors (i.e. when cluster has no quorum).",
 594                 type => 'boolean',
 595                 optional => 1,
 596             },
 597         },
 598     },
 599     returns => { type => 'null' },
 600     code => sub {
 601         my ($param) = @_;
 602
 603         PVE::Cluster::gen_pve_vzdump_symlink();
 604
 605         if (!PVE::Cluster::check_cfs_quorum(1)) {
 606             return undef if $param->{silent};
 607             die "no quorum - unable to update files\n";
 608         }
 609
 610         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
 611         PVE::Cluster::ssh_merge_keys();
 612         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
 613         PVE::Cluster::gen_pve_vzdump_files();
 614
 615         return undef;
 616     }});
 617
 618
 619 my $cmddef = {
 620     keygen => [ __PACKAGE__, 'keygen', ['filename']],
 621     create => [ __PACKAGE__, 'create', ['clustername']],
 622     add => [ __PACKAGE__, 'add', ['hostname']],
 623     addnode => [ __PACKAGE__, 'addnode', ['node']],
 624     delnode => [ __PACKAGE__, 'delnode', ['node']],
 625     status => [ __PACKAGE__, 'status' ],
 626     nodes => [ __PACKAGE__, 'nodes' ],
 627     expected => [ __PACKAGE__, 'expected', ['expected']],
 628     updatecerts => [ __PACKAGE__, 'updatecerts', []],
 629 };
 630
 631 my $cmd = shift;
 632
 633 if ($cmd && $cmd ne 'printmanpod' && $cmd ne 'verifyapi') {
 634     PVE::Cluster::check_cfs_is_mounted();
 635     PVE::Cluster::cfs_update();
 636 }
 637
 638 PVE::CLIHandler::handle_cmd($cmddef, "pvecm", $cmd, \@ARGV, undef, $0);
 639
 640 exit 0;
 641
 642 __END__
 643
 644 =head1 NAME
 645
 646 pvecm - Proxmox VE cluster manager toolkit
 647
 648 =head1 SYNOPSIS
 649
 650 =include synopsis
 651
 652 =head1 DESCRIPTION
 653
 654 pvecm is a program to manage the cluster configuration. It can be used
 655 to create a new cluster, join nodes to a cluster, leave the cluster,
 656 get status information and do various other cluster related tasks.
 657
 658 =include pve_copyright
 659
 660