data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EEXIST);
   6 use File::stat qw();
   7 use Socket;
   8 use Storable qw(dclone);
   9 use IO::File;
  10 use MIME::Base64;
  11 use XML::Parser;
  12 use Digest::SHA;
  13 use Digest::HMAC_SHA1;
  14 use PVE::Tools;
  15 use PVE::INotify;
  16 use PVE::IPCC;
  17 use PVE::SafeSyslog;
  18 use PVE::JSONSchema;
  19 use JSON;
  20 use RRDs;
  21 use Encode;
  22 use base 'Exporter';
  23
  24 our @EXPORT_OK = qw(
  25 cfs_read_file
  26 cfs_write_file
  27 cfs_register_file
  28 cfs_lock_file);
  29
  30 use Data::Dumper; # fixme: remove
  31
  32 # x509 certificate utils
  33
  34 my $basedir = "/etc/pve";
  35 my $authdir = "$basedir/priv";
  36 my $lockdir = "/etc/pve/priv/lock";
  37
  38 my $authprivkeyfn = "$authdir/authkey.key";
  39 my $authpubkeyfn = "$basedir/authkey.pub";
  40 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  41 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  42 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  43 # this is just a secret accessable by the web browser
  44 # and is used for CSRF prevention
  45 my $pvewww_key_fn = "$basedir/pve-www.key";
  46
  47 # ssh related files
  48 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  49 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  50 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  51 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  52 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  53 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  54 my $sshd_config_fn = "/etc/ssh/sshd_config";
  55 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  56 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  57 my $rootsshconfig = "/root/.ssh/config";
  58
  59 my $observed = {
  60     'vzdump.cron' => 1,
  61     'storage.cfg' => 1,
  62     'datacenter.cfg' => 1,
  63     'corosync.conf' => 1,
  64     'corosync.conf.new' => 1,
  65     'user.cfg' => 1,
  66     'domains.cfg' => 1,
  67     'priv/shadow.cfg' => 1,
  68     '/qemu-server/' => 1,
  69     '/openvz/' => 1,
  70     '/lxc/' => 1,
  71     'ha/crm_commands' => 1,
  72     'ha/manager_status' => 1,
  73     'ha/resources.cfg' => 1,
  74     'ha/groups.cfg' => 1,
  75     'ha/fence.cfg' => 1,
  76     'status.cfg' => 1,
  77 };
  78
  79 # only write output if something fails
  80 sub run_silent_cmd {
  81     my ($cmd) = @_;
  82
  83     my $outbuf = '';
  84
  85     my $record_output = sub {
  86         $outbuf .= shift;
  87         $outbuf .= "\n";
  88     };
  89
  90     eval {
  91         PVE::Tools::run_command($cmd, outfunc => $record_output,
  92                                 errfunc => $record_output);
  93     };
  94
  95     my $err = $@;
  96
  97     if ($err) {
  98         print STDERR $outbuf;
  99         die $err;
 100     }
 101 }
 102
 103 sub check_cfs_quorum {
 104     my ($noerr) = @_;
 105
 106     # note: -w filename always return 1 for root, so wee need
 107     # to use File::lstat here
 108     my $st = File::stat::lstat("$basedir/local");
 109     my $quorate = ($st && (($st->mode & 0200) != 0));
 110
 111     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 112
 113     return $quorate;
 114 }
 115
 116 sub check_cfs_is_mounted {
 117     my ($noerr) = @_;
 118
 119     my $res = -l "$basedir/local";
 120
 121     die "pve configuration filesystem not mounted\n"
 122         if !$res && !$noerr;
 123
 124     return $res;
 125 }
 126
 127 sub gen_local_dirs {
 128     my ($nodename) = @_;
 129
 130     check_cfs_is_mounted();
 131
 132     my @required_dirs = (
 133         "$basedir/priv",
 134         "$basedir/nodes",
 135         "$basedir/nodes/$nodename",
 136         "$basedir/nodes/$nodename/lxc",
 137         "$basedir/nodes/$nodename/qemu-server",
 138         "$basedir/nodes/$nodename/openvz",
 139         "$basedir/nodes/$nodename/priv");
 140
 141     foreach my $dir (@required_dirs) {
 142         if (! -d $dir) {
 143             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 144         }
 145     }
 146 }
 147
 148 sub gen_auth_key {
 149
 150     return if -f "$authprivkeyfn";
 151
 152     check_cfs_is_mounted();
 153
 154     mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
 155
 156     my $cmd = "openssl genrsa -out '$authprivkeyfn' 2048";
 157     run_silent_cmd($cmd);
 158
 159     $cmd = "openssl rsa -in '$authprivkeyfn' -pubout -out '$authpubkeyfn'";
 160     run_silent_cmd($cmd)
 161 }
 162
 163 sub gen_pveca_key {
 164
 165     return if -f $pveca_key_fn;
 166
 167     eval {
 168         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 169     };
 170
 171     die "unable to generate pve ca key:\n$@" if $@;
 172 }
 173
 174 sub gen_pveca_cert {
 175
 176     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 177         return 0;
 178     }
 179
 180     gen_pveca_key();
 181
 182     # we try to generate an unique 'subject' to avoid browser problems
 183     # (reused serial numbers, ..)
 184     my $nid = (split (/\s/, `md5sum '$pveca_key_fn'`))[0] || time();
 185
 186     eval {
 187         run_silent_cmd(['openssl', 'req', '-batch', '-days', '3650', '-new',
 188                         '-x509', '-nodes', '-key',
 189                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 190                         "/CN=Proxmox Virtual Environment/OU=$nid/O=PVE Cluster Manager CA/"]);
 191     };
 192
 193     die "generating pve root certificate failed:\n$@" if $@;
 194
 195     return 1;
 196 }
 197
 198 sub gen_pve_ssl_key {
 199     my ($nodename) = @_;
 200
 201     die "no node name specified" if !$nodename;
 202
 203     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 204
 205     return if -f $pvessl_key_fn;
 206
 207     eval {
 208         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 209     };
 210
 211     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 212 }
 213
 214 sub gen_pve_www_key {
 215
 216     return if -f $pvewww_key_fn;
 217
 218     eval {
 219         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 220     };
 221
 222     die "unable to generate pve www key:\n$@" if $@;
 223 }
 224
 225 sub update_serial {
 226     my ($serial) = @_;
 227
 228     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 229 }
 230
 231 sub gen_pve_ssl_cert {
 232     my ($force, $nodename, $ip) = @_;
 233
 234     die "no node name specified" if !$nodename;
 235     die "no IP specified" if !$ip;
 236
 237     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 238
 239     return if !$force && -f $pvessl_cert_fn;
 240
 241     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 242
 243     my $rc = PVE::INotify::read_file('resolvconf');
 244
 245     $names .= ",IP:$ip";
 246
 247     my $fqdn = $nodename;
 248
 249     $names .= ",DNS:$nodename";
 250
 251     if ($rc && $rc->{search}) {
 252         $fqdn = $nodename . "." . $rc->{search};
 253         $names .= ",DNS:$fqdn";
 254     }
 255
 256     my $sslconf = <<__EOD;
 257 RANDFILE = /root/.rnd
 258 extensions = v3_req
 259
 260 [ req ]
 261 default_bits = 2048
 262 distinguished_name = req_distinguished_name
 263 req_extensions = v3_req
 264 prompt = no
 265 string_mask = nombstr
 266
 267 [ req_distinguished_name ]
 268 organizationalUnitName = PVE Cluster Node
 269 organizationName = Proxmox Virtual Environment
 270 commonName = $fqdn
 271
 272 [ v3_req ]
 273 basicConstraints = CA:FALSE
 274 extendedKeyUsage = serverAuth
 275 subjectAltName = $names
 276 __EOD
 277
 278     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 279     my $fh = IO::File->new ($cfgfn, "w");
 280     print $fh $sslconf;
 281     close ($fh);
 282
 283     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 284     unlink $reqfn;
 285
 286     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 287     eval {
 288         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 289                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 290     };
 291
 292     if (my $err = $@) {
 293         unlink $reqfn;
 294         unlink $cfgfn;
 295         die "unable to generate pve certificate request:\n$err";
 296     }
 297
 298     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 299
 300     eval {
 301         run_silent_cmd(['openssl', 'x509', '-req', '-in', $reqfn, '-days', '3650',
 302                         '-out', $pvessl_cert_fn, '-CAkey', $pveca_key_fn,
 303                         '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn,
 304                         '-extfile', $cfgfn]);
 305     };
 306
 307     if (my $err = $@) {
 308         unlink $reqfn;
 309         unlink $cfgfn;
 310         die "unable to generate pve ssl certificate:\n$err";
 311     }
 312
 313     unlink $cfgfn;
 314     unlink $reqfn;
 315 }
 316
 317 sub gen_pve_node_files {
 318     my ($nodename, $ip, $opt_force) = @_;
 319
 320     gen_local_dirs($nodename);
 321
 322     gen_auth_key();
 323
 324     # make sure we have a (cluster wide) secret
 325     # for CSRFR prevention
 326     gen_pve_www_key();
 327
 328     # make sure we have a (per node) private key
 329     gen_pve_ssl_key($nodename);
 330
 331     # make sure we have a CA
 332     my $force = gen_pveca_cert();
 333
 334     $force = 1 if $opt_force;
 335
 336     gen_pve_ssl_cert($force, $nodename, $ip);
 337 }
 338
 339 my $vzdump_cron_dummy = <<__EOD;
 340 # cluster wide vzdump cron schedule
 341 # Atomatically generated file - do not edit
 342
 343 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 344
 345 __EOD
 346
 347 sub gen_pve_vzdump_symlink {
 348
 349     my $filename = "/etc/pve/vzdump.cron";
 350
 351     my $link_fn = "/etc/cron.d/vzdump";
 352
 353     if ((-f $filename) && (! -l $link_fn)) {
 354         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 355         symlink($filename, $link_fn);
 356     }
 357 }
 358
 359 sub gen_pve_vzdump_files {
 360
 361     my $filename = "/etc/pve/vzdump.cron";
 362
 363     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 364         if ! -f $filename;
 365
 366     gen_pve_vzdump_symlink();
 367 };
 368
 369 my $versions = {};
 370 my $vmlist = {};
 371 my $clinfo = {};
 372
 373 my $ipcc_send_rec = sub {
 374     my ($msgid, $data) = @_;
 375
 376     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 377
 378     die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
 379
 380     return $res;
 381 };
 382
 383 my $ipcc_send_rec_json = sub {
 384     my ($msgid, $data) = @_;
 385
 386     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 387
 388     die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
 389
 390     return decode_json($res);
 391 };
 392
 393 my $ipcc_get_config = sub {
 394     my ($path) = @_;
 395
 396     my $bindata = pack "Z*", $path;
 397     my $res = PVE::IPCC::ipcc_send_rec(6, $bindata);
 398     if (!defined($res)) {
 399         return undef if ($! != 0);
 400         return '';
 401     }
 402
 403     return $res;
 404 };
 405
 406 my $ipcc_get_status = sub {
 407     my ($name, $nodename) = @_;
 408
 409     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 410     return PVE::IPCC::ipcc_send_rec(5, $bindata);
 411 };
 412
 413 my $ipcc_update_status = sub {
 414     my ($name, $data) = @_;
 415
 416     my $raw = ref($data) ? encode_json($data) : $data;
 417     # update status
 418     my $bindata = pack "Z[256]Z*", $name, $raw;
 419
 420     return &$ipcc_send_rec(4, $bindata);
 421 };
 422
 423 my $ipcc_log = sub {
 424     my ($priority, $ident, $tag, $msg) = @_;
 425
 426     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 427     bytes::length($tag) + 1, $ident, $tag, $msg;
 428
 429     return &$ipcc_send_rec(7, $bindata);
 430 };
 431
 432 my $ipcc_get_cluster_log = sub {
 433     my ($user, $max) = @_;
 434
 435     $max = 0 if !defined($max);
 436
 437     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 438     return &$ipcc_send_rec(8, $bindata);
 439 };
 440
 441 my $ccache = {};
 442
 443 sub cfs_update {
 444     eval {
 445         my $res = &$ipcc_send_rec_json(1);
 446         #warn "GOT1: " . Dumper($res);
 447         die "no starttime\n" if !$res->{starttime};
 448
 449         if (!$res->{starttime} || !$versions->{starttime} ||
 450             $res->{starttime} != $versions->{starttime}) {
 451             #print "detected changed starttime\n";
 452             $vmlist = {};
 453             $clinfo = {};
 454             $ccache = {};
 455         }
 456
 457         $versions = $res;
 458     };
 459     my $err = $@;
 460     if ($err) {
 461         $versions = {};
 462         $vmlist = {};
 463         $clinfo = {};
 464         $ccache = {};
 465         warn $err;
 466     }
 467
 468     eval {
 469         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 470             #warn "detected new clinfo\n";
 471             $clinfo = &$ipcc_send_rec_json(2);
 472         }
 473     };
 474     $err = $@;
 475     if ($err) {
 476         $clinfo = {};
 477         warn $err;
 478     }
 479
 480     eval {
 481         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 482             #warn "detected new vmlist1\n";
 483             $vmlist = &$ipcc_send_rec_json(3);
 484         }
 485     };
 486     $err = $@;
 487     if ($err) {
 488         $vmlist = {};
 489         warn $err;
 490     }
 491 }
 492
 493 sub get_vmlist {
 494     return $vmlist;
 495 }
 496
 497 sub get_clinfo {
 498     return $clinfo;
 499 }
 500
 501 sub get_members {
 502     return $clinfo->{nodelist};
 503 }
 504
 505 sub get_nodelist {
 506
 507     my $nodelist = $clinfo->{nodelist};
 508
 509     my $result = [];
 510
 511     my $nodename = PVE::INotify::nodename();
 512
 513     if (!$nodelist || !$nodelist->{$nodename}) {
 514         return [ $nodename ];
 515     }
 516
 517     return [ keys %$nodelist ];
 518 }
 519
 520 sub broadcast_tasklist {
 521     my ($data) = @_;
 522
 523     eval {
 524         &$ipcc_update_status("tasklist", $data);
 525     };
 526
 527     warn $@ if $@;
 528 }
 529
 530 my $tasklistcache = {};
 531
 532 sub get_tasklist {
 533     my ($nodename) = @_;
 534
 535     my $kvstore = $versions->{kvstore} || {};
 536
 537     my $nodelist = get_nodelist();
 538
 539     my $res = [];
 540     foreach my $node (@$nodelist) {
 541         next if $nodename && ($nodename ne $node);
 542         eval {
 543             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 544             my $cd = $tasklistcache->{$node};
 545             if (!$cd || !$ver || !$cd->{version} ||
 546                 ($cd->{version} != $ver)) {
 547                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 548                 my $data = decode_json($raw);
 549                 push @$res, @$data;
 550                 $cd = $tasklistcache->{$node} = {
 551                     data => $data,
 552                     version => $ver,
 553                 };
 554             } elsif ($cd && $cd->{data}) {
 555                 push @$res, @{$cd->{data}};
 556             }
 557         };
 558         my $err = $@;
 559         syslog('err', $err) if $err;
 560     }
 561
 562     return $res;
 563 }
 564
 565 sub broadcast_rrd {
 566     my ($rrdid, $data) = @_;
 567
 568     eval {
 569         &$ipcc_update_status("rrd/$rrdid", $data);
 570     };
 571     my $err = $@;
 572
 573     warn $err if $err;
 574 }
 575
 576 my $last_rrd_dump = 0;
 577 my $last_rrd_data = "";
 578
 579 sub rrd_dump {
 580
 581     my $ctime = time();
 582
 583     my $diff = $ctime - $last_rrd_dump;
 584     if ($diff < 2) {
 585         return $last_rrd_data;
 586     }
 587
 588     my $raw;
 589     eval {
 590         $raw = &$ipcc_send_rec(10);
 591     };
 592     my $err = $@;
 593
 594     if ($err) {
 595         warn $err;
 596         return {};
 597     }
 598
 599     my $res = {};
 600
 601     if ($raw) {
 602         while ($raw =~ s/^(.*)\n//) {
 603             my ($key, @ela) = split(/:/, $1);
 604             next if !$key;
 605             next if !(scalar(@ela) > 1);
 606             $res->{$key} = \@ela;
 607         }
 608     }
 609
 610     $last_rrd_dump = $ctime;
 611     $last_rrd_data = $res;
 612
 613     return $res;
 614 }
 615
 616 sub create_rrd_data {
 617     my ($rrdname, $timeframe, $cf) = @_;
 618
 619     my $rrddir = "/var/lib/rrdcached/db";
 620
 621     my $rrd = "$rrddir/$rrdname";
 622
 623     my $setup = {
 624         hour =>  [ 60, 70 ],
 625         day  =>  [ 60*30, 70 ],
 626         week =>  [ 60*180, 70 ],
 627         month => [ 60*720, 70 ],
 628         year =>  [ 60*10080, 70 ],
 629     };
 630
 631     my ($reso, $count) = @{$setup->{$timeframe}};
 632     my $ctime  = $reso*int(time()/$reso);
 633     my $req_start = $ctime - $reso*$count;
 634
 635     $cf = "AVERAGE" if !$cf;
 636
 637     my @args = (
 638         "-s" => $req_start,
 639         "-e" => $ctime - 1,
 640         "-r" => $reso,
 641         );
 642
 643     my $socket = "/var/run/rrdcached.sock";
 644     push @args, "--daemon" => "unix:$socket" if -S $socket;
 645
 646     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 647
 648     my $err = RRDs::error;
 649     die "RRD error: $err\n" if $err;
 650
 651     die "got wrong time resolution ($step != $reso)\n"
 652         if $step != $reso;
 653
 654     my $res = [];
 655     my $fields = scalar(@$names);
 656     for my $line (@$data) {
 657         my $entry = { 'time' => $start };
 658         $start += $step;
 659         my $found_undefs;
 660         for (my $i = 0; $i < $fields; $i++) {
 661             my $name = $names->[$i];
 662             if (defined(my $val = $line->[$i])) {
 663                 $entry->{$name} = $val;
 664             } else {
 665                 # we only add entryies with all data defined
 666                 # extjs chart has problems with undefined values
 667                 $found_undefs = 1;
 668             }
 669         }
 670         push @$res, $entry if !$found_undefs;
 671     }
 672
 673     return $res;
 674 }
 675
 676 sub create_rrd_graph {
 677     my ($rrdname, $timeframe, $ds, $cf) = @_;
 678
 679     # Using RRD graph is clumsy - maybe it
 680     # is better to simply fetch the data, and do all display
 681     # related things with javascript (new extjs html5 graph library).
 682
 683     my $rrddir = "/var/lib/rrdcached/db";
 684
 685     my $rrd = "$rrddir/$rrdname";
 686
 687     my @ids = PVE::Tools::split_list($ds);
 688
 689     my $ds_txt = join('_', @ids);
 690
 691     my $filename = "${rrd}_${ds_txt}.png";
 692
 693     my $setup = {
 694         hour =>  [ 60, 60 ],
 695         day  =>  [ 60*30, 70 ],
 696         week =>  [ 60*180, 70 ],
 697         month => [ 60*720, 70 ],
 698         year =>  [ 60*10080, 70 ],
 699     };
 700
 701     my ($reso, $count) = @{$setup->{$timeframe}};
 702
 703     my @args = (
 704         "--imgformat" => "PNG",
 705         "--border" => 0,
 706         "--height" => 200,
 707         "--width" => 800,
 708         "--start" => - $reso*$count,
 709         "--end" => 'now' ,
 710         "--lower-limit" => 0,
 711         );
 712
 713     my $socket = "/var/run/rrdcached.sock";
 714     push @args, "--daemon" => "unix:$socket" if -S $socket;
 715
 716     my @coldef = ('#00ddff', '#ff0000');
 717
 718     $cf = "AVERAGE" if !$cf;
 719
 720     my $i = 0;
 721     foreach my $id (@ids) {
 722         my $col = $coldef[$i++] || die "fixme: no color definition";
 723         push @args, "DEF:${id}=$rrd:${id}:$cf";
 724         my $dataid = $id;
 725         if ($id eq 'cpu' || $id eq 'iowait') {
 726             push @args, "CDEF:${id}_per=${id},100,*";
 727             $dataid = "${id}_per";
 728         }
 729         push @args, "LINE2:${dataid}${col}:${id}";
 730     }
 731
 732     push @args, '--full-size-mode';
 733
 734     # we do not really store data into the file
 735     my $res = RRDs::graphv('', @args);
 736
 737     my $err = RRDs::error;
 738     die "RRD error: $err\n" if $err;
 739
 740     return { filename => $filename, image => $res->{image} };
 741 }
 742
 743 # a fast way to read files (avoid fuse overhead)
 744 sub get_config {
 745     my ($path) = @_;
 746
 747     return &$ipcc_get_config($path);
 748 }
 749
 750 sub get_cluster_log {
 751     my ($user, $max) = @_;
 752
 753     return &$ipcc_get_cluster_log($user, $max);
 754 }
 755
 756 my $file_info = {};
 757
 758 sub cfs_register_file {
 759     my ($filename, $parser, $writer) = @_;
 760
 761     $observed->{$filename} || die "unknown file '$filename'";
 762
 763     die "file '$filename' already registered" if $file_info->{$filename};
 764
 765     $file_info->{$filename} = {
 766         parser => $parser,
 767         writer => $writer,
 768     };
 769 }
 770
 771 my $ccache_read = sub {
 772     my ($filename, $parser, $version) = @_;
 773
 774     $ccache->{$filename} = {} if !$ccache->{$filename};
 775
 776     my $ci = $ccache->{$filename};
 777
 778     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 779         # we always call the parser, even when the file does not exists
 780         # (in that case $data is undef)
 781         my $data = get_config($filename);
 782         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 783         $ci->{version} = $version;
 784     }
 785
 786     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 787
 788     return $res;
 789 };
 790
 791 sub cfs_file_version {
 792     my ($filename) = @_;
 793
 794     my $version;
 795     my $infotag;
 796     if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
 797         my ($type, $vmid) = ($1, $2);
 798         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 799             $version = $vmlist->{ids}->{$vmid}->{version};
 800         }
 801         $infotag = "/$type/";
 802     } else {
 803         $infotag = $filename;
 804         $version = $versions->{$filename};
 805     }
 806
 807     my $info = $file_info->{$infotag} ||
 808         die "unknown file type '$filename'\n";
 809
 810     return wantarray ? ($version, $info) : $version;
 811 }
 812
 813 sub cfs_read_file {
 814     my ($filename) = @_;
 815
 816     my ($version, $info) = cfs_file_version($filename);
 817     my $parser = $info->{parser};
 818
 819     return &$ccache_read($filename, $parser, $version);
 820 }
 821
 822 sub cfs_write_file {
 823     my ($filename, $data) = @_;
 824
 825     my ($version, $info) = cfs_file_version($filename);
 826
 827     my $writer = $info->{writer} || die "no writer defined";
 828
 829     my $fsname = "/etc/pve/$filename";
 830
 831     my $raw = &$writer($fsname, $data);
 832
 833     if (my $ci = $ccache->{$filename}) {
 834         $ci->{version} = undef;
 835     }
 836
 837     PVE::Tools::file_set_contents($fsname, $raw);
 838 }
 839
 840 my $cfs_lock = sub {
 841     my ($lockid, $timeout, $code, @param) = @_;
 842
 843     my $res;
 844
 845     # this timeout is for aquire the lock
 846     $timeout = 10 if !$timeout;
 847
 848     my $filename = "$lockdir/$lockid";
 849
 850     my $msg = "can't aquire cfs lock '$lockid'";
 851
 852     eval {
 853
 854         mkdir $lockdir;
 855
 856         if (! -d $lockdir) {
 857             die "$msg: pve cluster filesystem not online.\n";
 858         }
 859
 860         local $SIG{ALRM} = sub { die "got lock request timeout\n"; };
 861
 862         alarm ($timeout);
 863
 864         if (!(mkdir $filename)) {
 865             print STDERR "trying to aquire cfs lock '$lockid' ...";
 866             while (1) {
 867                 if (!(mkdir $filename)) {
 868                     (utime 0, 0, $filename); # cfs unlock request
 869                 } else {
 870                     print STDERR " OK\n";
 871                     last;
 872                 }
 873                 sleep(1);
 874             }
 875         }
 876
 877         # fixed command timeout: cfs locks have a timeout of 120
 878         # using 60 gives us another 60 seconds to abort the task
 879         alarm(60);
 880         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 881
 882         cfs_update(); # make sure we read latest versions inside code()
 883
 884         $res = &$code(@param);
 885
 886         alarm(0);
 887     };
 888
 889     my $err = $@;
 890
 891     alarm(0);
 892
 893     if ($err && ($err eq "got lock request timeout\n") &&
 894         !check_cfs_quorum()){
 895         $err = "$msg: no quorum!\n";
 896     }
 897
 898     if (!$err || $err !~ /^got lock timeout -/) {
 899         rmdir $filename; # cfs unlock
 900     }
 901
 902     if ($err) {
 903         $@ = $err;
 904         return undef;
 905     }
 906
 907     $@ = undef;
 908
 909     return $res;
 910 };
 911
 912 sub cfs_lock_file {
 913     my ($filename, $timeout, $code, @param) = @_;
 914
 915     my $info = $observed->{$filename} || die "unknown file '$filename'";
 916
 917     my $lockid = "file-$filename";
 918     $lockid =~ s/[.\/]/_/g;
 919
 920     &$cfs_lock($lockid, $timeout, $code, @param);
 921 }
 922
 923 sub cfs_lock_storage {
 924     my ($storeid, $timeout, $code, @param) = @_;
 925
 926     my $lockid = "storage-$storeid";
 927
 928     &$cfs_lock($lockid, $timeout, $code, @param);
 929 }
 930
 931 sub cfs_lock_domain {
 932     my ($domainname, $timeout, $code, @param) = @_;
 933
 934     my $lockid = "domain-$domainname";
 935
 936     &$cfs_lock($lockid, $timeout, $code, @param);
 937 }
 938
 939 my $log_levels = {
 940     "emerg" => 0,
 941     "alert" => 1,
 942     "crit" => 2,
 943     "critical" => 2,
 944     "err" => 3,
 945     "error" => 3,
 946     "warn" => 4,
 947     "warning" => 4,
 948     "notice" => 5,
 949     "info" => 6,
 950     "debug" => 7,
 951 };
 952
 953 sub log_msg {
 954    my ($priority, $ident, $msg) = @_;
 955
 956    if (my $tmp = $log_levels->{$priority}) {
 957        $priority = $tmp;
 958    }
 959
 960    die "need numeric log priority" if $priority !~ /^\d+$/;
 961
 962    my $tag = PVE::SafeSyslog::tag();
 963
 964    $msg = "empty message" if !$msg;
 965
 966    $ident = "" if !$ident;
 967    $ident = encode("ascii", $ident,
 968                    sub { sprintf "\\u%04x", shift });
 969
 970    my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
 971
 972    if ($ident) {
 973        syslog($priority, "<%s> %s", $ident, $ascii);
 974    } else {
 975        syslog($priority, "%s", $ascii);
 976    }
 977
 978    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
 979
 980    syslog("err", "writing cluster log failed: $@") if $@;
 981 }
 982
 983 sub check_vmid_unused {
 984     my ($vmid, $noerr) = @_;
 985
 986     my $vmlist = get_vmlist();
 987
 988     my $d = $vmlist->{ids}->{$vmid};
 989     return 1 if !defined($d);
 990
 991     return undef if $noerr;
 992
 993     my $vmtypestr =  $d->{type} eq 'qemu' ? 'VM' : 'CT';
 994     die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
 995 }
 996
 997 sub check_node_exists {
 998     my ($nodename, $noerr) = @_;
 999
1000     my $nodelist = $clinfo->{nodelist};
1001     return 1 if $nodelist && $nodelist->{$nodename};
1002
1003     return undef if $noerr;
1004
1005     die "no such cluster node '$nodename'\n";
1006 }
1007
1008 # this is also used to get the IP of the local node
1009 sub remote_node_ip {
1010     my ($nodename, $noerr) = @_;
1011
1012     my $nodelist = $clinfo->{nodelist};
1013     if ($nodelist && $nodelist->{$nodename}) {
1014         if (my $ip = $nodelist->{$nodename}->{ip}) {
1015             return $ip if !wantarray;
1016             my $family = $nodelist->{$nodename}->{address_family};
1017             if (!$family) {
1018                 $nodelist->{$nodename}->{address_family} =
1019                     $family =
1020                     PVE::Tools::get_host_address_family($ip);
1021             }
1022             return ($ip, $family);
1023         }
1024     }
1025
1026     # fallback: try to get IP by other means
1027     my ($family, $packed_ip);
1028
1029     eval {
1030         my @res = PVE::Tools::getaddrinfo_all($nodename);
1031         $family = $res[0]->{family};
1032         $packed_ip = (PVE::Tools::unpack_sockaddr_in46($res[0]->{addr}))[2];
1033     };
1034
1035     if ($@) {
1036         die "hostname lookup failed:\n$@" if !$noerr;
1037         return undef;
1038     }
1039
1040     my $ip = Socket::inet_ntop($family, $packed_ip);
1041     if ($ip =~ m/^127\.|^::1$/) {
1042         die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr;
1043         return undef;
1044     }
1045
1046     return wantarray ? ($ip, $family) : $ip;
1047 }
1048
1049 # ssh related utility functions
1050
1051 sub ssh_merge_keys {
1052     # remove duplicate keys in $sshauthkeys
1053     # ssh-copy-id simply add keys, so the file can grow to large
1054
1055     my $data = '';
1056     if (-f $sshauthkeys) {
1057         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
1058         chomp($data);
1059     }
1060
1061     my $found_backup;
1062     if (-f $rootsshauthkeysbackup) {
1063         $data .= "\n";
1064         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
1065         chomp($data);
1066         $found_backup = 1;
1067     }
1068
1069     # always add ourself
1070     if (-f $ssh_rsa_id) {
1071         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
1072         chomp($pub);
1073         $data .= "\n$pub\n";
1074     }
1075
1076     my $newdata = "";
1077     my $vhash = {};
1078     my @lines = split(/\n/, $data);
1079     foreach my $line (@lines) {
1080         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1081             next if $vhash->{$3}++;
1082         }
1083         $newdata .= "$line\n";
1084     }
1085
1086     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
1087
1088     if ($found_backup && -l $rootsshauthkeys) {
1089         # everything went well, so we can remove the backup
1090         unlink $rootsshauthkeysbackup;
1091     }
1092 }
1093
1094 sub setup_sshd_config {
1095
1096     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
1097
1098     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1099
1100     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1101         chomp $conf;
1102         $conf .= "\nPermitRootLogin yes\n";
1103     }
1104
1105     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
1106
1107     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
1108 }
1109
1110 sub setup_rootsshconfig {
1111
1112     # create ssh key if it does not exist
1113     if (! -f $ssh_rsa_id) {
1114         mkdir '/root/.ssh/';
1115         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1116     }
1117
1118     # create ssh config if it does not exist
1119     if (! -f $rootsshconfig) {
1120         mkdir '/root/.ssh';
1121         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
1122             # this is the default ciphers list from debian openssl0.9.8 except blowfish is added as prefered
1123             print $fh "Ciphers blowfish-cbc,aes128-ctr,aes192-ctr,aes256-ctr,arcfour256,arcfour128,aes128-cbc,3des-cbc\n";
1124             close($fh);
1125         }
1126     }
1127 }
1128
1129 sub setup_ssh_keys {
1130
1131     mkdir $authdir;
1132
1133     my $import_ok;
1134
1135     if (! -f $sshauthkeys) {
1136         my $old;
1137         if (-f $rootsshauthkeys) {
1138             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
1139         }
1140         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1141             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
1142             close($fh);
1143             $import_ok = 1;
1144         }
1145     }
1146
1147     warn "can't create shared ssh key database '$sshauthkeys'\n"
1148         if ! -f $sshauthkeys;
1149
1150     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
1151         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1152             warn "rename $rootsshauthkeys failed - $!\n";
1153         }
1154     }
1155
1156     if (! -l $rootsshauthkeys) {
1157         symlink $sshauthkeys, $rootsshauthkeys;
1158     }
1159
1160     if (! -l $rootsshauthkeys) {
1161         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1162     } else {
1163         unlink $rootsshauthkeysbackup if $import_ok;
1164     }
1165 }
1166
1167 sub ssh_unmerge_known_hosts {
1168     return if ! -l $sshglobalknownhosts;
1169
1170     my $old = '';
1171     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1172         if -f $sshknownhosts;
1173
1174     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1175 }
1176
1177 sub ssh_merge_known_hosts {
1178     my ($nodename, $ip_address, $createLink) = @_;
1179
1180     die "no node name specified" if !$nodename;
1181     die "no ip address specified" if !$ip_address;
1182
1183     mkdir $authdir;
1184
1185     if (! -f $sshknownhosts) {
1186         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1187             close($fh);
1188         }
1189     }
1190
1191     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1192
1193     my $new = '';
1194
1195     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1196         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1197     }
1198
1199     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1200     # Note: file sometimes containe emty lines at start, so we use multiline match
1201     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
1202     $hostkey = $1;
1203
1204     my $data = '';
1205     my $vhash = {};
1206
1207     my $found_nodename;
1208     my $found_local_ip;
1209
1210     my $merge_line = sub {
1211         my ($line, $all) = @_;
1212
1213         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1214             my $key = $1;
1215             my $rsakey = $2;
1216             if (!$vhash->{$key}) {
1217                 $vhash->{$key} = 1;
1218                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1219                     my $salt = decode_base64($1);
1220                     my $digest = $2;
1221                     my $hmac = Digest::HMAC_SHA1->new($salt);
1222                     $hmac->add($nodename);
1223                     my $hd = $hmac->b64digest . '=';
1224                     if ($digest eq $hd) {
1225                         if ($rsakey eq $hostkey) {
1226                             $found_nodename = 1;
1227                             $data .= $line;
1228                         }
1229                         return;
1230                     }
1231                     $hmac = Digest::HMAC_SHA1->new($salt);
1232                     $hmac->add($ip_address);
1233                     $hd = $hmac->b64digest . '=';
1234                     if ($digest eq $hd) {
1235                         if ($rsakey eq $hostkey) {
1236                             $found_local_ip = 1;
1237                             $data .= $line;
1238                         }
1239                         return;
1240                     }
1241                 }
1242                 $data .= $line;
1243             }
1244         } elsif ($all) {
1245             $data .= $line;
1246         }
1247     };
1248
1249     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1250         my $line = "$2\n";
1251         next if $line =~ m/^\s*$/; # skip empty lines
1252         next if $line =~ m/^#/; # skip comments
1253         &$merge_line($line, 1);
1254     }
1255
1256     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1257         my $line = "$2\n";
1258         next if $line =~ m/^\s*$/; # skip empty lines
1259         next if $line =~ m/^#/; # skip comments
1260         &$merge_line($line);
1261     }
1262
1263     my $addIndex = $$;
1264     my $add_known_hosts_entry  = sub {
1265         my ($name, $hostkey) = @_;
1266         $addIndex++;
1267         my $hmac = Digest::HMAC_SHA1->new("$addIndex" . time());
1268         my $b64salt = $hmac->b64digest . '=';
1269         $hmac = Digest::HMAC_SHA1->new(decode_base64($b64salt));
1270         $hmac->add($name);
1271         my $digest = $hmac->b64digest . '=';
1272         $data .= "|1|$b64salt|$digest $hostkey\n";
1273     };
1274
1275     if (!$found_nodename || !$found_local_ip) {
1276         &$add_known_hosts_entry($nodename, $hostkey) if !$found_nodename;
1277         &$add_known_hosts_entry($ip_address, $hostkey) if !$found_local_ip;
1278     }
1279
1280     PVE::Tools::file_set_contents($sshknownhosts, $data);
1281
1282     return if !$createLink;
1283
1284     unlink $sshglobalknownhosts;
1285     symlink $sshknownhosts, $sshglobalknownhosts;
1286
1287     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1288         if ! -l $sshglobalknownhosts;
1289
1290 }
1291
1292 my $datacenter_schema = {
1293     type => "object",
1294     additionalProperties => 0,
1295     properties => {
1296         keyboard => {
1297             optional => 1,
1298             type => 'string',
1299             description => "Default keybord layout for vnc server.",
1300             enum => PVE::Tools::kvmkeymaplist(),
1301         },
1302         language => {
1303             optional => 1,
1304             type => 'string',
1305             description => "Default GUI language.",
1306             enum => [ 'en', 'de' ],
1307         },
1308         http_proxy => {
1309             optional => 1,
1310             type => 'string',
1311             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1312             pattern => "http://.*",
1313         },
1314         migration_unsecure => {
1315             optional => 1,
1316             type => 'boolean',
1317             description => "Migration is secure using SSH tunnel by default. For secure private networks you can disable it to speed up migration.",
1318         },
1319         console => {
1320             optional => 1,
1321             type => 'string',
1322             description => "Select the default Console viewer. You can either use the builtin java applet (VNC), an external virt-viewer comtatible application (SPICE), or an HTML5 based viewer (noVNC).",
1323             enum => ['applet', 'vv', 'html5'],
1324         },
1325         email_from => {
1326             optional => 1,
1327             type => 'string',
1328             format => 'email-opt',
1329             description => "Specify email address to send notification from (default is root@\$hostname)",
1330         },
1331         max_workers => {
1332             optional => 1,
1333             type => 'integer',
1334             minimum => 1,
1335             description => "Defines how many workers (per node) are maximal started ".
1336               " on actions like 'stopall VMs' or task from the ha-manager.",
1337         },
1338         fencing => {
1339             optional => 1,
1340             type => 'string',
1341             default => 'watchdog',
1342             enum => [ 'watchdog', 'hardware', 'both' ],
1343             description => "Set the fencing mode of the HA cluster. Hardware mode " .
1344               "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
1345               " With both all two modes are used. " .
1346               " NOTE: 'hardware' and 'both' are EXPERIMENTAL & WIP",
1347         },
1348     },
1349 };
1350
1351 # make schema accessible from outside (for documentation)
1352 sub get_datacenter_schema { return $datacenter_schema };
1353
1354 sub parse_datacenter_config {
1355     my ($filename, $raw) = @_;
1356
1357     return PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
1358 }
1359
1360 sub write_datacenter_config {
1361     my ($filename, $cfg) = @_;
1362
1363     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1364 }
1365
1366 cfs_register_file('datacenter.cfg',
1367                   \&parse_datacenter_config,
1368                   \&write_datacenter_config);
1369
1370 # a very simply parser ...
1371 sub parse_corosync_conf {
1372     my ($filename, $raw) = @_;
1373
1374     return {} if !$raw;
1375
1376     my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
1377
1378     $raw =~ s/#.*$//mg;
1379     $raw =~ s/\r?\n/ /g;
1380     $raw =~ s/\s+/ /g;
1381     $raw =~ s/^\s+//;
1382     $raw =~ s/\s*$//;
1383
1384     my @tokens = split(/\s/, $raw);
1385
1386     my $conf = { section => 'main', children => [] };
1387
1388     my $stack = [];
1389     my $section = $conf;
1390
1391     while (defined(my $token = shift @tokens)) {
1392         my $nexttok = $tokens[0];
1393
1394         if ($nexttok && ($nexttok eq '{')) {
1395             shift @tokens; # skip '{'
1396             my $new_section = {
1397                 section => $token,
1398                 children => [],
1399             };
1400             push @{$section->{children}}, $new_section;
1401             push @$stack, $section;
1402             $section = $new_section;
1403             next;
1404         }
1405
1406         if ($token eq '}') {
1407             $section = pop @$stack;
1408             die "parse error - uncexpected '}'\n" if !$section;
1409             next;
1410         }
1411
1412         my $key = $token;
1413         die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
1414
1415         die "parse error - no value for '$key'\n" if !defined($nexttok);
1416         my $value = shift @tokens;
1417
1418         push @{$section->{children}}, { key => $key, value => $value };
1419     }
1420
1421     $conf->{digest} = $digest;
1422
1423     return $conf;
1424 }
1425
1426 my $dump_corosync_section;
1427 $dump_corosync_section = sub {
1428     my ($section, $prefix) = @_;
1429
1430     my $raw = $prefix . $section->{section} . " {\n";
1431
1432     my @list = grep { defined($_->{key}) } @{$section->{children}};
1433     foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
1434         $raw .= $prefix . "  $child->{key}: $child->{value}\n";
1435     }
1436
1437     @list = grep { defined($_->{section}) } @{$section->{children}};
1438     foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
1439         $raw .= &$dump_corosync_section($child, "$prefix  ");
1440     }
1441
1442     $raw .= $prefix . "}\n\n";
1443
1444     return $raw;
1445
1446 };
1447
1448 sub write_corosync_conf {
1449     my ($filename, $conf) = @_;
1450
1451     my $raw = '';
1452
1453     my $prefix = '';
1454
1455     die "no main section" if $conf->{section} ne 'main';
1456
1457     my @list = grep { defined($_->{key}) } @{$conf->{children}};
1458     foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
1459         $raw .= "$child->{key}: $child->{value}\n";
1460     }
1461
1462     @list = grep { defined($_->{section}) } @{$conf->{children}};
1463     foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
1464         $raw .= &$dump_corosync_section($child, $prefix);
1465     }
1466
1467     return $raw;
1468 }
1469
1470 sub corosync_conf_version {
1471     my ($conf, $noerr, $new_value) = @_;
1472
1473     foreach my $child (@{$conf->{children}}) {
1474         next if !defined($child->{section});
1475         if ($child->{section} eq 'totem') {
1476             foreach my $e (@{$child->{children}}) {
1477                 next if !defined($e->{key});
1478                 if ($e->{key} eq 'config_version') {
1479                     if ($new_value) {
1480                         $e->{value} = $new_value;
1481                         return $new_value;
1482                     } elsif (my $version = int($e->{value})) {
1483                         return $version;
1484                     }
1485                     last;
1486                 }
1487             }
1488         }
1489     }
1490
1491     return undef if $noerr;
1492
1493     die "invalid corosync config - unable to read version\n";
1494 }
1495
1496 # read only - use "rename corosync.conf.new corosync.conf" to write
1497 PVE::Cluster::cfs_register_file('corosync.conf', \&parse_corosync_conf);
1498 # this is read/write
1499 PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_corosync_conf,
1500                                 \&write_corosync_conf);
1501
1502 # bash completion helpers
1503
1504 sub complete_next_vmid {
1505
1506     my $vmlist = get_vmlist() || {};
1507     my $idlist = $vmlist->{ids} || {};
1508
1509     for (my $i = 100; $i < 10000; $i++) {
1510         return [$i] if !defined($idlist->{$i});
1511     }
1512
1513     return [];
1514 }
1515
1516 sub complete_vmid {
1517
1518     my $vmlist = get_vmlist();
1519     my $ids = $vmlist->{ids} || {};
1520
1521     return [ keys %$ids ];
1522 }
1523
1524 sub complete_local_vmid {
1525
1526     my $vmlist = get_vmlist();
1527     my $ids = $vmlist->{ids} || {};
1528
1529     my $nodename = PVE::INotify::nodename();
1530
1531     my $res = [];
1532     foreach my $vmid (keys %$ids) {
1533         my $d = $ids->{$vmid};
1534         next if !$d->{node} || $d->{node} ne $nodename;
1535         push @$res, $vmid;
1536     }
1537
1538     return $res;
1539 }
1540
1541 sub complete_migration_target {
1542
1543     my $res = [];
1544
1545     my $nodename = PVE::INotify::nodename();
1546
1547     my $nodelist = get_nodelist();
1548     foreach my $node (@$nodelist) {
1549         next if $node eq $nodename;
1550         push @$res, $node;
1551     }
1552
1553     return $res;
1554 }
1555
1556 1;