data/PVE/pvecm

   1 #!/usr/bin/perl -w
   2 use strict;
   3 use Getopt::Long;
   4 use Socket;
   5 use IO::File;
   6 use File::Path;
   7 use File::Basename;
   8 use Data::Dumper; # fixme: remove
   9 use PVE::Tools;
  10 use PVE::Cluster;
  11 use PVE::INotify;
  12 use PVE::JSONSchema;
  13 use PVE::CLIHandler;
  14
  15 use base qw(PVE::CLIHandler);
  16
  17 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
  18 $ENV{LC_ALL} = 'C';
  19 $ENV{HOME} = '/root'; # for ssh-copy-id
  20
  21 die "please run as root\n" if $> != 0;
  22
  23 my $nodename = PVE::INotify::nodename();
  24 # trigger check that we have resolvable name
  25 my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
  26
  27 my $basedir = "/etc/pve";
  28 my $clusterconf = "$basedir/cluster.conf";
  29 my $libdir = "/var/lib/pve-cluster";
  30 my $backupdir = "/var/lib/pve-cluster/backup";
  31 my $dbfile = "$libdir/config.db";
  32 my $authfile = "$libdir/corosync.authkey";
  33
  34
  35 sub backup_database {
  36
  37     print "backup old database\n";
  38
  39     mkdir $backupdir;
  40
  41     my $ctime = time();
  42     my $cmd = "echo '.dump' |";
  43     $cmd .= "sqlite3 '$dbfile' |";
  44     $cmd .= "gzip - >'${backupdir}/config-${ctime}.sql.gz'";
  45
  46     system($cmd) == 0 ||
  47         die "can't backup old database: $!\n";
  48
  49     # purge older backup
  50     my $maxfiles = 10;
  51
  52     my @bklist = ();
  53     foreach my $fn (<$backupdir/config-*.sql.gz>) {
  54         if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
  55             push @bklist, [$fn, $1];
  56         }
  57     }
  58
  59     @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
  60
  61     while (scalar (@bklist) >= $maxfiles) {
  62         my $d = pop @bklist;
  63         print "delete old backup '$d->[0]'\n";
  64         unlink $d->[0];
  65     }
  66 }
  67
  68 __PACKAGE__->register_method ({
  69     name => 'keygen',
  70     path => 'keygen',
  71     method => 'PUT',
  72     description => "Generate new cryptographic key for corosync.",
  73     parameters => {
  74         additionalProperties => 0,
  75         properties => {
  76             filename => {
  77                 type => 'string',
  78                 description => "Output file name"
  79             }
  80         },
  81     },
  82     returns => { type => 'null' },
  83
  84     code => sub {
  85         my ($param) = @_;
  86
  87         my $filename = $param->{filename};
  88
  89         # test EUID
  90         $> == 0 || die "Error: Authorization key must be generated as root user.\n";
  91         my $dirname = dirname($filename);
  92         my $basename = basename($filename);
  93
  94         File::Path::make_path($dirname) if $dirname;
  95
  96         my $fh = IO::File->new ("/dev/urandom", 'r') ||
  97             die "can't open /dev/urandom - $!\n";
  98
  99         my $keysize = 128;
 100         my $key = '';
 101
 102         my $bytes = 0;
 103         while ($bytes < $keysize) {
 104             my $rb = sysread($fh, $key, $keysize - $bytes, $bytes);
 105             ($rb > 0) || die "Could not read /dev/urandom - $!\n";
 106             $bytes += $rb;
 107         }
 108
 109         close($fh);
 110
 111         my $tmpfn = "$filename.tmp.$$";
 112         $fh = IO::File->new ($tmpfn, O_CREAT|O_WRONLY, 0400);
 113         die "can't open temporary file '$tmpfn' - $!\n" if !$fh;
 114
 115         eval {
 116             my $wb;
 117             (($wb = syswrite($fh, $key)) == $keysize) ||
 118                 die "writing key failed - short write $wb\n;"
 119         };
 120         my $err = $@;
 121
 122         $fh->close();
 123
 124         if ($err) {
 125             unlink $tmpfn;
 126             die $err;
 127         }
 128
 129         if (!rename($tmpfn, $filename)) {
 130             my $err = $!;
 131             unlink $tmpfn;
 132             die "rename $tmpfn to $filename failed - $err\n";
 133         }
 134
 135         return undef;
 136     }});
 137
 138 __PACKAGE__->register_method ({
 139     name => 'create',
 140     path => 'create',
 141     method => 'PUT',
 142     description => "Generate new cluster configuration.",
 143     parameters => {
 144         additionalProperties => 0,
 145         properties => {
 146             clustername => {
 147                 description => "The name of the cluster.",
 148                 type => 'string', format => 'pve-node',
 149                 maxLength => 15,
 150             },
 151             nodeid => {
 152                 type => 'integer',
 153                 description => "Node id for this node.",
 154                 minimum => 1,
 155                 optional => 1,
 156             },
 157             votes => {
 158                 type => 'integer',
 159                 description => "Number of votes for this node",
 160                 minimum => 1,
 161                 optional => 1,
 162             },
 163         },
 164     },
 165     returns => { type => 'null' },
 166
 167     code => sub {
 168         my ($param) = @_;
 169
 170         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
 171
 172         PVE::Cluster::setup_rootsshconfig();
 173         PVE::Cluster::setup_ssh_keys();
 174
 175         -f $authfile || __PACKAGE__->keygen({filename => $authfile});
 176
 177         -f $authfile || die "no authentication key available\n";
 178
 179         my $clustername = $param->{clustername};
 180
 181         $param->{nodeid} = 1 if !$param->{nodeid};
 182
 183         $param->{votes} = 1 if !defined($param->{votes});
 184
 185         my $config = <<_EOD;
 186 <?xml version="1.0"?>
 187 <cluster name="$clustername" config_version="1">
 188
 189   <cman keyfile="$authfile">
 190   </cman>
 191
 192   <clusternodes>
 193   <clusternode name="${nodename}" votes="$param->{votes}" nodeid="$param->{nodeid}"/>
 194   </clusternodes>
 195
 196 </cluster>
 197 _EOD
 198 ;
 199         PVE::Tools::file_set_contents($clusterconf, $config);
 200
 201         PVE::Cluster::ssh_merge_keys();
 202
 203         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 204
 205         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 206
 207         PVE::Tools::run_command('/etc/init.d/pve-cluster restart'); # restart
 208
 209         # that cman init script returns strange values - simply ignore for now
 210         system('/etc/init.d/cman start');
 211
 212         # also start clvm
 213         system('/etc/init.d/clvm start');
 214
 215         return undef;
 216 }});
 217
 218 __PACKAGE__->register_method ({
 219     name => 'addnode',
 220     path => 'addnode',
 221     method => 'PUT',
 222     description => "Adds a node to the cluster configuration.",
 223     parameters => {
 224         additionalProperties => 0,
 225         properties => {
 226             node => PVE::JSONSchema::get_standard_option('pve-node'),
 227             nodeid => {
 228                 type => 'integer',
 229                 description => "Node id for this node.",
 230                 minimum => 1,
 231                 optional => 1,
 232             },
 233             votes => {
 234                 type => 'integer',
 235                 description => "Number of votes for this node",
 236                 minimum => 0,
 237                 optional => 1,
 238             },
 239             force => {
 240                 type => 'boolean',
 241                 description => "Do not throw error if node already exists.",
 242                 optional => 1,
 243             },
 244         },
 245     },
 246     returns => { type => 'null' },
 247
 248     code => sub {
 249         my ($param) = @_;
 250
 251         PVE::Cluster::check_cfs_quorum();
 252
 253         my $lst = lsnode();
 254
 255         my $name = $param->{node};
 256
 257         if (defined(my $res = $lst->{$name})) {
 258             $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 259             $param->{votes} = $res->{votes} if !defined($param->{votes});
 260
 261             if ($res->{votes} == $param->{votes} &&
 262                 $res->{nodeid} == $param->{nodeid}) {
 263                 print "node $name already defined\n";
 264                 if ($param->{force}) {
 265                     exit (0);
 266                 } else {
 267                     exit (-1);
 268                 }
 269             }
 270         } elsif (!$param->{nodeid}) {
 271             my $nodeid = 1;
 272
 273             while(1) {
 274                 my $found = 0;
 275                 foreach my $v (values %$lst) {
 276                     if ($v->{nodeid} eq $nodeid) {
 277                         $found = 1;
 278                         $nodeid++;
 279                         last;
 280                     }
 281                 }
 282                 last if !$found;
 283             };
 284
 285             $param->{nodeid} = $nodeid;
 286         }
 287
 288         $param->{votes} = 1 if !defined($param->{votes});
 289
 290         PVE::Cluster::gen_local_dirs($name);
 291
 292         eval {  PVE::Cluster::ssh_merge_keys(); };
 293         warn $@ if $@;
 294
 295         my $cmd = ['ccs_tool', 'addnode', '-c', $clusterconf];
 296
 297         # NOTE: cman does not like votes="0"
 298         if ($param->{votes}) {
 299             push @$cmd, '-v', $param->{votes};
 300         }
 301
 302         push @$cmd, '-n', $param->{nodeid}, $name;
 303
 304         system(@$cmd) == 0 || exit(-1);
 305
 306         exit (0);
 307     }});
 308
 309
 310 __PACKAGE__->register_method ({
 311     name => 'delnode',
 312     path => 'delnode',
 313     method => 'PUT',
 314     description => "Removes a node to the cluster configuration.",
 315     parameters => {
 316         additionalProperties => 0,
 317         properties => {
 318             node => PVE::JSONSchema::get_standard_option('pve-node'),
 319         },
 320     },
 321     returns => { type => 'null' },
 322
 323     code => sub {
 324         my ($param) = @_;
 325
 326         PVE::Cluster::check_cfs_quorum();
 327
 328         my $cmd = ['ccs_tool', 'delnode', '-c', $clusterconf, $param->{node}];
 329
 330         exec (@$cmd);
 331
 332         exit (-1); # should not be reached
 333     }});
 334
 335 __PACKAGE__->register_method ({
 336     name => 'add',
 337     path => 'add',
 338     method => 'PUT',
 339     description => "Adds the current node to an existing cluster.",
 340     parameters => {
 341         additionalProperties => 0,
 342         properties => {
 343             hostname => {
 344                 type => 'string',
 345                 description => "Hostname (or IP) of an existing cluster member."
 346             },
 347             nodeid => {
 348                 type => 'integer',
 349                 description => "Node id for this node.",
 350                 minimum => 1,
 351                 optional => 1,
 352             },
 353             votes => {
 354                 type => 'integer',
 355                 description => "Number of votes for this node",
 356                 minimum => 0,
 357                 optional => 1,
 358             },
 359             force => {
 360                 type => 'boolean',
 361                 description => "Do not throw error if node already exists.",
 362                 optional => 1,
 363             },
 364         },
 365     },
 366     returns => { type => 'null' },
 367
 368     code => sub {
 369         my ($param) = @_;
 370
 371         PVE::Cluster::setup_rootsshconfig();
 372         PVE::Cluster::setup_ssh_keys();
 373
 374         my $host = $param->{hostname};
 375
 376         if (!$param->{force}) {
 377
 378             if (-f $authfile) {
 379                 die "authentication key already exists\n";
 380             }
 381
 382             if (-f $clusterconf)  {
 383                 die "cluster config '$clusterconf' already exists\n";
 384             }
 385
 386             my $vmlist = PVE::Cluster::get_vmlist();
 387             if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 388                 die "this host already contains virtual machines - please remove the first\n";
 389             }
 390
 391             if (system("cman_tool status >/dev/null 2>&1") == 0) {
 392                 die "cman is already running\n";
 393             }
 394         }
 395
 396         # make sure known_hosts is on local filesystem
 397         PVE::Cluster::ssh_unmerge_known_hosts();
 398
 399         my $cmd = "ssh-copy-id -i /root/.ssh/id_rsa 'root\@$host' >/dev/null 2>&1";
 400         system ($cmd) == 0 ||
 401             die "unable to copy ssh ID\n";
 402
 403         $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
 404                 'pvecm', 'addnode', $nodename, '--force', 1];
 405
 406         push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
 407
 408         push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
 409
 410         if (system (@$cmd) != 0) {
 411             my $cmdtxt = join (' ', @$cmd);
 412             die "unable to add node: command failed ($cmdtxt)\n";
 413         }
 414
 415         my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
 416         mkdir $tmpdir;
 417
 418         eval {
 419             print "copy corosync auth key\n";
 420             $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
 421                     "$host:$authfile $clusterconf", $tmpdir];
 422
 423             system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
 424
 425             mkdir "/etc/cluster";
 426             my $confbase = basename($clusterconf);
 427
 428             $cmd = "cp '$tmpdir/$confbase' '/etc/cluster/$confbase'";
 429             system($cmd) == 0 || die "can't copy cluster configuration\n";
 430
 431             my $keybase = basename($authfile);
 432             system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
 433                 die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
 434
 435             print "stopping pve-cluster service\n";
 436
 437             system("umount $basedir -f >/dev/null 2>&1");
 438             system("/etc/init.d/pve-cluster stop") == 0 ||
 439                 die "can't stop pve-cluster service\n";
 440
 441             backup_database();
 442
 443             unlink $dbfile;
 444
 445             system("/etc/init.d/pve-cluster start") == 0 ||
 446                 die "starting pve-cluster failed\n";
 447
 448             system("/etc/init.d/cman start");
 449
 450             # wait for quorum
 451             my $printqmsg = 1;
 452             while (!PVE::Cluster::check_cfs_quorum(1)) {
 453                 if ($printqmsg) {
 454                     print "waiting for quorum...";
 455                     STDOUT->flush();
 456                     $printqmsg = 0;
 457                 }
 458                 sleep(1);
 459             }
 460             print "OK\n" if !$printqmsg;
 461
 462             system("/etc/init.d/clvm start");
 463
 464             print "generating node certificates\n";
 465             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 466
 467             print "merge known_hosts file\n";
 468             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 469
 470             print "restart services\n";
 471             # restart pvedaemon (changed certs)
 472             system("/etc/init.d/pvedaemon restart");
 473             # restart pveproxy (changed certs)
 474             system("/etc/init.d/pveproxy restart");
 475
 476             print "successfully added node '$nodename' to cluster.\n";
 477         };
 478         my $err = $@;
 479
 480         rmtree $tmpdir;
 481
 482         die $err if $err;
 483
 484         return undef;
 485     }});
 486
 487 __PACKAGE__->register_method ({
 488     name => 'status',
 489     path => 'status',
 490     method => 'GET',
 491     description => "Displays the local view of the cluster status.",
 492     parameters => {
 493         additionalProperties => 0,
 494         properties => {},
 495     },
 496     returns => { type => 'null' },
 497
 498     code => sub {
 499         my ($param) = @_;
 500
 501         my $cmd = ['cman_tool', 'status'];
 502
 503         exec (@$cmd);
 504
 505         exit (-1); # should not be reached
 506     }});
 507
 508 __PACKAGE__->register_method ({
 509     name => 'nodes',
 510     path => 'nodes',
 511     method => 'GET',
 512     description => "Displays the local view of the cluster nodes.",
 513     parameters => {
 514         additionalProperties => 0,
 515         properties => {},
 516     },
 517     returns => { type => 'null' },
 518
 519     code => sub {
 520         my ($param) = @_;
 521
 522         my $cmd = ['cman_tool', 'nodes'];
 523
 524         exec (@$cmd);
 525
 526         exit (-1); # should not be reached
 527     }});
 528
 529 __PACKAGE__->register_method ({
 530     name => 'expected',
 531     path => 'expected',
 532     method => 'PUT',
 533     description => "Tells CMAN a new value of expected votes.",
 534     parameters => {
 535         additionalProperties => 0,
 536         properties => {
 537             expected => {
 538                 type => 'integer',
 539                 description => "Expected votes",
 540                 minimum => 1,
 541             },
 542         },
 543     },
 544     returns => { type => 'null' },
 545
 546     code => sub {
 547         my ($param) = @_;
 548
 549         my $cmd = ['cman_tool', 'expected', '-e', $param->{expected}];
 550
 551         exec (@$cmd);
 552
 553         exit (-1); # should not be reached
 554
 555     }});
 556
 557 sub lsnode {
 558
 559     my $res = {};
 560
 561     my $cmd = ['ccs_tool',  'lsnode', '-c', $clusterconf];
 562
 563     my $scan = 0;
 564     my $parser = sub {
 565         my $line = shift;
 566
 567         if ($line =~ m/^Nodename\s+Votes\s+Nodeid\s/i) {
 568             $scan = 1;
 569             return;
 570         }
 571         return if !$scan;
 572         if ($line =~ m/^(\S+)\s+(\d+)\s+(\d+)\s/) {
 573             $res->{$1} = {
 574                 name => $1,
 575                 votes => $2,
 576                 nodeid => $3 };
 577         }
 578     };
 579
 580     PVE::Tools::run_command($cmd, outfunc => $parser);
 581     return $res;
 582 }
 583
 584 __PACKAGE__->register_method ({
 585     name => 'updatecerts',
 586     path => 'updatecerts',
 587     method => 'PUT',
 588     description => "Update node certificates (and generate all needed files/directories).",
 589     parameters => {
 590         additionalProperties => 0,
 591         properties => {
 592             force => {
 593                 description => "Force generation of new SSL certifate.",
 594                 type => 'boolean',
 595                 optional => 1,
 596             },
 597             silent => {
 598                 description => "Ignore errors (i.e. when cluster has no quorum).",
 599                 type => 'boolean',
 600                 optional => 1,
 601             },
 602         },
 603     },
 604     returns => { type => 'null' },
 605     code => sub {
 606         my ($param) = @_;
 607
 608         PVE::Cluster::setup_rootsshconfig();
 609
 610         PVE::Cluster::gen_pve_vzdump_symlink();
 611
 612         if (!PVE::Cluster::check_cfs_quorum(1)) {
 613             return undef if $param->{silent};
 614             die "no quorum - unable to update files\n";
 615         }
 616
 617         PVE::Cluster::setup_ssh_keys();
 618
 619         PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
 620         PVE::Cluster::ssh_merge_keys();
 621         PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
 622         PVE::Cluster::gen_pve_vzdump_files();
 623
 624         return undef;
 625     }});
 626
 627
 628 my $cmddef = {
 629     keygen => [ __PACKAGE__, 'keygen', ['filename']],
 630     create => [ __PACKAGE__, 'create', ['clustername']],
 631     add => [ __PACKAGE__, 'add', ['hostname']],
 632     addnode => [ __PACKAGE__, 'addnode', ['node']],
 633     delnode => [ __PACKAGE__, 'delnode', ['node']],
 634     status => [ __PACKAGE__, 'status' ],
 635     nodes => [ __PACKAGE__, 'nodes' ],
 636     expected => [ __PACKAGE__, 'expected', ['expected']],
 637     updatecerts => [ __PACKAGE__, 'updatecerts', []],
 638 };
 639
 640 my $cmd = shift;
 641
 642 if ($cmd && $cmd ne 'printmanpod' && $cmd ne 'verifyapi') {
 643     PVE::Cluster::check_cfs_is_mounted();
 644     PVE::Cluster::cfs_update();
 645 }
 646
 647 PVE::CLIHandler::handle_cmd($cmddef, "pvecm", $cmd, \@ARGV, undef, $0);
 648
 649 exit 0;
 650
 651 __END__
 652
 653 =head1 NAME
 654
 655 pvecm - Proxmox VE cluster manager toolkit
 656
 657 =head1 SYNOPSIS
 658
 659 =include synopsis
 660
 661 =head1 DESCRIPTION
 662
 663 pvecm is a program to manage the cluster configuration. It can be used
 664 to create a new cluster, join nodes to a cluster, leave the cluster,
 665 get status information and do various other cluster related tasks.
 666
 667 =include pve_copyright
 668
 669