data/PVE/pvecm

   1 #!/usr/bin/perl -w
   2 use strict;
   3 use Getopt::Long;
   4 use Socket;
   5 use IO::File;
   6 use File::Path;
   7 use File::Basename;
   8 use Data::Dumper; # fixme: remove
   9 use PVE::Tools;
  10 use PVE::Cluster;
  11 use PVE::INotify;
  12 use PVE::JSONSchema;
  13 use PVE::CLIHandler;
  14
  15 use base qw(PVE::CLIHandler);
  16
  17 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
  18
  19 die "please run as root\n" if $> != 0;
  20
  21 my $nodename = PVE::INotify::nodename();
  22 # trigger check that we have resolvable name
  23 my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
  24
  25 my $basedir = "/etc/pve";
  26 my $clusterconf = "$basedir/cluster.conf";
  27 my $libdir = "/var/lib/pve-cluster";
  28 my $backupdir = "/var/lib/pve-cluster/backup";
  29 my $dbfile = "$libdir/config.db";
  30 my $authfile = "$libdir/corosync.authkey";
  31
  32
  33 sub backup_database {
  34
  35     print "backup old database\n";
  36
  37     mkdir $backupdir;
  38
  39     my $ctime = time();
  40     my $cmd = "echo '.dump' |";
  41     $cmd .= "sqlite3 '$dbfile' |";
  42     $cmd .= "gzip - >'${backupdir}/config-${ctime}.sql.gz'";
  43
  44     system($cmd) == 0 ||
  45         die "can't backup old database: $!\n";
  46
  47     # purge older backup
  48     my $maxfiles = 10;
  49
  50     my @bklist = ();
  51     foreach my $fn (<$backupdir/config-*.sql.gz>) {
  52         if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
  53             push @bklist, [$fn, $1];
  54         }
  55     }
  56
  57     @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
  58
  59     while (scalar (@bklist) >= $maxfiles) {
  60         my $d = pop @bklist;
  61         print "delete old backup '$d->[0]'\n";
  62         unlink $d->[0];
  63     }
  64 }
  65
  66 __PACKAGE__->register_method ({
  67     name => 'keygen',
  68     path => 'keygen',
  69     method => 'PUT',
  70     description => "Generate new cryptographic key for corosync.",
  71     parameters => {
  72         additionalProperties => 0,
  73         properties => {
  74             filename => {
  75                 type => 'string',
  76                 description => "Output file name"
  77             }
  78         },
  79     },
  80     returns => { type => 'null' },
  81
  82     code => sub {
  83         my ($param) = @_;
  84
  85         my $filename = $param->{filename};
  86
  87         # test EUID
  88         $> == 0 || die "Error: Authorization key must be generated as root user.\n";
  89         my $dirname = dirname($filename);
  90         my $basename = basename($filename);
  91
  92         File::Path::make_path($dirname) if $dirname;
  93
  94         my $fh = IO::File->new ("/dev/urandom", 'r') ||
  95             die "can't open /dev/urandom - $!\n";
  96
  97         my $keysize = 128;
  98         my $key = '';
  99
 100         my $bytes = 0;
 101         while ($bytes < $keysize) {
 102             my $rb = sysread($fh, $key, $keysize - $bytes, $bytes);
 103             ($rb > 0) || die "Could not read /dev/urandom - $!\n";
 104             $bytes += $rb;
 105         }
 106
 107         close($fh);
 108
 109         my $tmpfn = "$filename.tmp.$$";
 110         $fh = IO::File->new ($tmpfn, O_CREAT|O_WRONLY, 0400);
 111         die "can't open temporary file '$tmpfn' - $!\n" if !$fh;
 112
 113         eval {
 114             my $wb;
 115             (($wb = syswrite($fh, $key)) == $keysize) ||
 116                 die "writing key failed - short write $wb\n;"
 117         };
 118         my $err = $@;
 119
 120         $fh->close();
 121
 122         if ($err) {
 123             unlink $tmpfn;
 124             die $err;
 125         }
 126
 127         if (!rename($tmpfn, $filename)) {
 128             my $err = $!;
 129             unlink $tmpfn;
 130             die "rename $tmpfn to $filename failed - $err\n";
 131         }
 132
 133         return undef;
 134     }});
 135
 136 __PACKAGE__->register_method ({
 137     name => 'create',
 138     path => 'create',
 139     method => 'PUT',
 140     description => "Generate new cluster configuration.",
 141     parameters => {
 142         additionalProperties => 0,
 143         properties => {
 144             clustername => {
 145                 description => "The name of the cluster.",
 146                 type => 'string', format => 'pve-node',
 147                 maxLength => 15,
 148             },
 149             nodeid => {
 150                 type => 'integer',
 151                 description => "Node id for this node.",
 152                 minimum => 1,
 153                 optional => 1,
 154             },
 155             votes => {
 156                 type => 'integer',
 157                 description => "Number of votes for this node",
 158                 minimum => 1,
 159                 optional => 1,
 160             },
 161         },
 162     },
 163     returns => { type => 'null' },
 164
 165     code => sub {
 166         my ($param) = @_;
 167
 168         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
 169
 170         PVE::Cluster::setup_ssh_keys();
 171
 172         -f $authfile || __PACKAGE__->keygen({filename => $authfile});
 173
 174         -f $authfile || die "no authentication key available\n";
 175
 176         my $clustername = $param->{clustername};
 177
 178         $param->{nodeid} = 1 if !$param->{nodeid};
 179
 180         $param->{votes} = 1 if !defined($param->{votes});
 181
 182         my $config = <<_EOD;
 183 <?xml version="1.0"?>
 184 <cluster name="$clustername" config_version="1">
 185
 186   <cman keyfile="$authfile">
 187   </cman>
 188
 189   <clusternodes>
 190   <clusternode name="${nodename}" votes="$param->{votes}" nodeid="$param->{nodeid}"/>
 191   </clusternodes>
 192
 193 </cluster>
 194 _EOD
 195 ;
 196         PVE::Tools::file_set_contents($clusterconf, $config);
 197
 198         PVE::Cluster::ssh_merge_keys();
 199
 200         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 201
 202         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 203
 204         PVE::Tools::run_command('/etc/init.d/pve-cluster restart'); # restart
 205
 206         # that cman init script returns strange values - simply ignore for now
 207         system('/etc/init.d/cman start');
 208
 209         # also start clvm
 210         system('/etc/init.d/clvm start');
 211
 212         return undef;
 213 }});
 214
 215 __PACKAGE__->register_method ({
 216     name => 'addnode',
 217     path => 'addnode',
 218     method => 'PUT',
 219     description => "Adds a node to the cluster configuration.",
 220     parameters => {
 221         additionalProperties => 0,
 222         properties => {
 223             node => PVE::JSONSchema::get_standard_option('pve-node'),
 224             nodeid => {
 225                 type => 'integer',
 226                 description => "Node id for this node.",
 227                 minimum => 1,
 228                 optional => 1,
 229             },
 230             votes => {
 231                 type => 'integer',
 232                 description => "Number of votes for this node",
 233                 minimum => 0,
 234                 optional => 1,
 235             },
 236             force => {
 237                 type => 'boolean',
 238                 description => "Do not throw error if node already exists.",
 239                 optional => 1,
 240             },
 241         },
 242     },
 243     returns => { type => 'null' },
 244
 245     code => sub {
 246         my ($param) = @_;
 247
 248         PVE::Cluster::check_cfs_quorum();
 249
 250         my $lst = lsnode();
 251
 252         my $name = $param->{node};
 253
 254         if (defined(my $res = $lst->{$name})) {
 255             $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 256             $param->{votes} = $res->{votes} if !defined($param->{votes});
 257
 258             if ($res->{votes} == $param->{votes} &&
 259                 $res->{nodeid} == $param->{nodeid}) {
 260                 print "node $name already defined\n";
 261                 if ($param->{force}) {
 262                     exit (0);
 263                 } else {
 264                     exit (-1);
 265                 }
 266             }
 267         } elsif (!$param->{nodeid}) {
 268             my $nodeid = 1;
 269
 270             while(1) {
 271                 my $found = 0;
 272                 foreach my $v (values %$lst) {
 273                     if ($v->{nodeid} eq $nodeid) {
 274                         $found = 1;
 275                         $nodeid++;
 276                         last;
 277                     }
 278                 }
 279                 last if !$found;
 280             };
 281
 282             $param->{nodeid} = $nodeid;
 283         }
 284
 285         $param->{votes} = 1 if !defined($param->{votes});
 286
 287         PVE::Cluster::gen_local_dirs($name);
 288
 289         eval {  PVE::Cluster::ssh_merge_keys(); };
 290         warn $@ if $@;
 291
 292         my $cmd = ['ccs_tool', 'addnode', '-c', $clusterconf];
 293
 294         # NOTE: cman does not like votes="0"
 295         if ($param->{votes}) {
 296             push @$cmd, '-v', $param->{votes};
 297         }
 298
 299         push @$cmd, '-n', $param->{nodeid}, $name;
 300
 301         system(@$cmd) == 0 || exit(-1);
 302
 303         exit (0);
 304     }});
 305
 306
 307 __PACKAGE__->register_method ({
 308     name => 'delnode',
 309     path => 'delnode',
 310     method => 'PUT',
 311     description => "Removes a node to the cluster configuration.",
 312     parameters => {
 313         additionalProperties => 0,
 314         properties => {
 315             node => PVE::JSONSchema::get_standard_option('pve-node'),
 316         },
 317     },
 318     returns => { type => 'null' },
 319
 320     code => sub {
 321         my ($param) = @_;
 322
 323         PVE::Cluster::check_cfs_quorum();
 324
 325         my $cmd = ['ccs_tool', 'delnode', '-c', $clusterconf, $param->{node}];
 326
 327         exec (@$cmd);
 328
 329         exit (-1); # should not be reached
 330     }});
 331
 332 __PACKAGE__->register_method ({
 333     name => 'add',
 334     path => 'add',
 335     method => 'PUT',
 336     description => "Adds the current node to an existing cluster.",
 337     parameters => {
 338         additionalProperties => 0,
 339         properties => {
 340             hostname => {
 341                 type => 'string',
 342                 description => "Hostname (or IP) of an existing cluster member."
 343             },
 344             nodeid => {
 345                 type => 'integer',
 346                 description => "Node id for this node.",
 347                 minimum => 1,
 348                 optional => 1,
 349             },
 350             votes => {
 351                 type => 'integer',
 352                 description => "Number of votes for this node",
 353                 minimum => 0,
 354                 optional => 1,
 355             },
 356             force => {
 357                 type => 'boolean',
 358                 description => "Do not throw error if node already exists.",
 359                 optional => 1,
 360             },
 361         },
 362     },
 363     returns => { type => 'null' },
 364
 365     code => sub {
 366         my ($param) = @_;
 367
 368         PVE::Cluster::setup_ssh_keys();
 369
 370         my $host = $param->{hostname};
 371
 372         if (!$param->{force}) {
 373
 374             if (-f $authfile) {
 375                 die "authentication key already exists\n";
 376             }
 377
 378             if (-f $clusterconf)  {
 379                 die "cluster config '$clusterconf' already exists\n";
 380             }
 381
 382             my $vmlist = PVE::Cluster::get_vmlist();
 383             if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 384                 die "this host already contains virtual machines - please remove the first\n";
 385             }
 386
 387             if (system("cman_tool status >/dev/null 2>&1") == 0) {
 388                 die "cman is already running\n";
 389             }
 390         }
 391
 392         # make sure known_hosts is on local filesystem
 393         PVE::Cluster::ssh_unmerge_known_hosts();
 394
 395         my $cmd = "ssh-copy-id 'root\@$host' >/dev/null 2>&1";
 396         system ($cmd) == 0 ||
 397             die "unable to copy ssh ID\n";
 398
 399         $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
 400                 'pvecm', 'addnode', $nodename, '--force', 1];
 401
 402         push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
 403
 404         push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
 405
 406         if (system (@$cmd) != 0) {
 407             my $cmdtxt = join (' ', @$cmd);
 408             die "unable to add node: command failed ($cmdtxt)\n";
 409         }
 410
 411         my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
 412         mkdir $tmpdir;
 413
 414         eval {
 415             print "copy corosync auth key\n";
 416             $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
 417                     "$host:$authfile $clusterconf", $tmpdir];
 418
 419             system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
 420
 421             mkdir "/etc/cluster";
 422             my $confbase = basename($clusterconf);
 423
 424             $cmd = "cp '$tmpdir/$confbase' '/etc/cluster/$confbase'";
 425             system($cmd) == 0 || die "can't copy cluster configuration\n";
 426
 427             my $keybase = basename($authfile);
 428             system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
 429                 die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
 430
 431             print "stopping pve-cluster service\n";
 432
 433             system("umount $basedir -f >/dev/null 2>&1");
 434             system("/etc/init.d/pve-cluster stop") == 0 ||
 435                 die "can't stop pve-cluster service\n";
 436
 437             backup_database();
 438
 439             unlink $dbfile;
 440
 441             system("/etc/init.d/pve-cluster start") == 0 ||
 442                 die "starting pve-cluster failed\n";
 443
 444             system("/etc/init.d/cman start");
 445
 446             # wait for quorum
 447             my $printqmsg = 1;
 448             while (!PVE::Cluster::check_cfs_quorum(1)) {
 449                 if ($printqmsg) {
 450                     print "waiting for quorum...";
 451                     STDOUT->flush();
 452                     $printqmsg = 0;
 453                 }
 454                 sleep(1);
 455             }
 456             print "OK\n" if !$printqmsg;
 457
 458             system("/etc/init.d/clvm start");
 459
 460             print "generating node certificates\n";
 461             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 462
 463             print "merge known_hosts file\n";
 464             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 465
 466             print "restart services\n";
 467             # restart pvedaemon (changed certs)
 468             system("/etc/init.d/pvedaemon restart");
 469             # restart apache (changed certs)
 470             system("/etc/init.d/apache2 restart");
 471
 472             print "successfully added node '$nodename' to cluster.\n";
 473         };
 474         my $err = $@;
 475
 476         rmtree $tmpdir;
 477
 478         die $err if $err;
 479
 480         return undef;
 481     }});
 482
 483 __PACKAGE__->register_method ({
 484     name => 'status',
 485     path => 'status',
 486     method => 'GET',
 487     description => "Displays the local view of the cluster status.",
 488     parameters => {
 489         additionalProperties => 0,
 490         properties => {},
 491     },
 492     returns => { type => 'null' },
 493
 494     code => sub {
 495         my ($param) = @_;
 496
 497         my $cmd = ['cman_tool', 'status'];
 498
 499         exec (@$cmd);
 500
 501         exit (-1); # should not be reached
 502     }});
 503
 504 __PACKAGE__->register_method ({
 505     name => 'nodes',
 506     path => 'nodes',
 507     method => 'GET',
 508     description => "Displays the local view of the cluster nodes.",
 509     parameters => {
 510         additionalProperties => 0,
 511         properties => {},
 512     },
 513     returns => { type => 'null' },
 514
 515     code => sub {
 516         my ($param) = @_;
 517
 518         my $cmd = ['cman_tool', 'nodes'];
 519
 520         exec (@$cmd);
 521
 522         exit (-1); # should not be reached
 523     }});
 524
 525 __PACKAGE__->register_method ({
 526     name => 'expected',
 527     path => 'expected',
 528     method => 'PUT',
 529     description => "Tells CMAN a new value of expected votes.",
 530     parameters => {
 531         additionalProperties => 0,
 532         properties => {
 533             expected => {
 534                 type => 'integer',
 535                 description => "Expected votes",
 536                 minimum => 1,
 537             },
 538         },
 539     },
 540     returns => { type => 'null' },
 541
 542     code => sub {
 543         my ($param) = @_;
 544
 545         my $cmd = ['cman_tool', 'expected', '-e', $param->{expected}];
 546
 547         exec (@$cmd);
 548
 549         exit (-1); # should not be reached
 550
 551     }});
 552
 553 sub lsnode {
 554
 555     my $res = {};
 556
 557     my $cmd = ['ccs_tool',  'lsnode', '-c', $clusterconf];
 558
 559     my $scan = 0;
 560     my $parser = sub {
 561         my $line = shift;
 562
 563         if ($line =~ m/^Nodename\s+Votes\s+Nodeid\s/i) {
 564             $scan = 1;
 565             return;
 566         }
 567         return if !$scan;
 568         if ($line =~ m/^(\S+)\s+(\d+)\s+(\d+)\s/) {
 569             $res->{$1} = {
 570                 name => $1,
 571                 votes => $2,
 572                 nodeid => $3 };
 573         }
 574     };
 575
 576     PVE::Tools::run_command($cmd, outfunc => $parser);
 577     return $res;
 578 }
 579
 580 __PACKAGE__->register_method ({
 581     name => 'updatecerts',
 582     path => 'updatecerts',
 583     method => 'PUT',
 584     description => "Update node certificates (and generate all needed files/directories).",
 585     parameters => {
 586         additionalProperties => 0,
 587         properties => {
 588             force => {
 589                 description => "Force generation of new SSL certifate.",
 590                 type => 'boolean',
 591                 optional => 1,
 592             },
 593             silent => {
 594                 description => "Ignore errors (i.e. when cluster has no quorum).",
 595                 type => 'boolean',
 596                 optional => 1,
 597             },
 598         },
 599     },
 600     returns => { type => 'null' },
 601     code => sub {
 602         my ($param) = @_;
 603
 604         PVE::Cluster::gen_pve_vzdump_symlink();
 605
 606         if (!PVE::Cluster::check_cfs_quorum(1)) {
 607             return undef if $param->{silent};
 608             die "no quorum - unable to update files\n";
 609         }
 610
 611         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
 612         PVE::Cluster::ssh_merge_keys();
 613         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
 614         PVE::Cluster::gen_pve_vzdump_files();
 615
 616         return undef;
 617     }});
 618
 619
 620 my $cmddef = {
 621     keygen => [ __PACKAGE__, 'keygen', ['filename']],
 622     create => [ __PACKAGE__, 'create', ['clustername']],
 623     add => [ __PACKAGE__, 'add', ['hostname']],
 624     addnode => [ __PACKAGE__, 'addnode', ['node']],
 625     delnode => [ __PACKAGE__, 'delnode', ['node']],
 626     status => [ __PACKAGE__, 'status' ],
 627     nodes => [ __PACKAGE__, 'nodes' ],
 628     expected => [ __PACKAGE__, 'expected', ['expected']],
 629     updatecerts => [ __PACKAGE__, 'updatecerts', []],
 630 };
 631
 632 my $cmd = shift;
 633
 634 if ($cmd && $cmd ne 'printmanpod' && $cmd ne 'verifyapi') {
 635     PVE::Cluster::check_cfs_is_mounted();
 636     PVE::Cluster::cfs_update();
 637 }
 638
 639 PVE::CLIHandler::handle_cmd($cmddef, "pvecm", $cmd, \@ARGV, undef, $0);
 640
 641 exit 0;
 642
 643 __END__
 644
 645 =head1 NAME
 646
 647 pvecm - Proxmox VE cluster manager toolkit
 648
 649 =head1 SYNOPSIS
 650
 651 =include synopsis
 652
 653 =head1 DESCRIPTION
 654
 655 pvecm is a program to manage the cluster configuration. It can be used
 656 to create a new cluster, join nodes to a cluster, leave the cluster,
 657 get status information and do various other cluster related tasks.
 658
 659 =include pve_copyright
 660
 661