data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use POSIX;
   5 use File::stat qw();
   6 use Socket;
   7 use Storable qw(dclone);
   8 use IO::File;
   9 use MIME::Base64;
  10 use Digest::HMAC_SHA1;
  11 use PVE::Tools;
  12 use PVE::INotify;
  13 use PVE::IPCC;
  14 use PVE::SafeSyslog;
  15 use JSON;
  16 use RRDs;
  17 use Encode;
  18 use base 'Exporter';
  19
  20 our @EXPORT_OK = qw(
  21 cfs_read_file
  22 cfs_write_file
  23 cfs_register_file
  24 cfs_lock_file);
  25
  26 use Data::Dumper; # fixme: remove
  27
  28 # x509 certificate utils
  29
  30 my $basedir = "/etc/pve";
  31 my $authdir = "$basedir/priv";
  32 my $lockdir = "/etc/pve/priv/lock";
  33
  34 my $authprivkeyfn = "$authdir/authkey.key";
  35 my $authpubkeyfn = "$basedir/authkey.pub";
  36 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  37 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  38 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  39 # this is just a secret accessable by the web browser
  40 # and is used for CSRF prevention
  41 my $pvewww_key_fn = "$basedir/pve-www.key";
  42
  43 # ssh related files
  44 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  45 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  46 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  47 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  48 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  49 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  50 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  51
  52 my $observed = {
  53     'storage.cfg' => 1,
  54     'datacenter.cfg' => 1,
  55     'cluster.cfg' => 1,
  56     'user.cfg' => 1,
  57     'domains.cfg' => 1,
  58     'priv/shadow.cfg' => 1,
  59     '/qemu-server/' => 1,
  60     '/openvz/' => 1,
  61 };
  62
  63 # only write output if something fails
  64 sub run_silent_cmd {
  65     my ($cmd) = @_;
  66
  67     my $outbuf = '';
  68
  69     my $record_output = sub {
  70         $outbuf .= shift;
  71         $outbuf .= "\n";
  72     };
  73
  74     eval {
  75         PVE::Tools::run_command($cmd, outfunc => $record_output,
  76                                 errfunc => $record_output);
  77     };
  78
  79     my $err = $@;
  80
  81     if ($err) {
  82         print STDERR $outbuf;
  83         die $err;
  84     }
  85 }
  86
  87 sub check_cfs_quorum {
  88     my ($noerr) = @_;
  89
  90     # note: -w filename always return 1 for root, so wee need
  91     # to use File::lstat here
  92     my $st = File::stat::lstat("$basedir/local");
  93     my $quorate = ($st && (($st->mode & 0200) != 0));
  94
  95     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
  96
  97     return $quorate;
  98 }
  99
 100 sub check_cfs_is_mounted {
 101     my ($noerr) = @_;
 102
 103     my $res = -l "$basedir/local";
 104
 105     die "pve configuration filesystem not mounted\n"
 106         if !$res && !$noerr;
 107
 108     return $res;
 109 }
 110
 111 sub gen_local_dirs {
 112     my ($nodename) = @_;
 113
 114     check_cfs_is_mounted();
 115
 116     my @required_dirs = (
 117         "$basedir/priv",
 118         "$basedir/nodes",
 119         "$basedir/nodes/$nodename",
 120         "$basedir/nodes/$nodename/qemu-server",
 121         "$basedir/nodes/$nodename/openvz",
 122         "$basedir/nodes/$nodename/priv");
 123
 124     foreach my $dir (@required_dirs) {
 125         if (! -d $dir) {
 126             mkdir($dir) || die "unable to create directory '$dir' - $!\n";
 127         }
 128     }
 129 }
 130
 131 sub gen_auth_key {
 132
 133     return if -f "$authprivkeyfn";
 134
 135     check_cfs_is_mounted();
 136
 137     mkdir $authdir || die "unable to create dir '$authdir' - $!\n";
 138
 139     my $cmd = "openssl genrsa -out '$authprivkeyfn' 2048";
 140     run_silent_cmd($cmd);
 141
 142     $cmd = "openssl rsa -in '$authprivkeyfn' -pubout -out '$authpubkeyfn'";
 143     run_silent_cmd($cmd)
 144 }
 145
 146 sub gen_pveca_key {
 147
 148     return if -f $pveca_key_fn;
 149
 150     eval {
 151         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '2048']);
 152     };
 153
 154     die "unable to generate pve ca key:\n$@" if $@;
 155 }
 156
 157 sub gen_pveca_cert {
 158
 159     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 160         return 0;
 161     }
 162
 163     gen_pveca_key();
 164
 165     # we try to generate an unique 'subject' to avoid browser problems
 166     # (reused serial numbers, ..)
 167     my $nid = (split (/\s/, `md5sum '$pveca_key_fn'`))[0] || time();
 168
 169     eval {
 170         run_silent_cmd(['openssl', 'req', '-batch', '-days', '3650', '-new',
 171                         '-x509', '-nodes', '-key',
 172                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 173                         "/CN=Proxmox Virtual Environment/OU=$nid/O=PVE Cluster Manager CA/"]);
 174     };
 175
 176     die "generating pve root certificate failed:\n$@" if $@;
 177
 178     return 1;
 179 }
 180
 181 sub gen_pve_ssl_key {
 182     my ($nodename) = @_;
 183
 184     die "no node name specified" if !$nodename;
 185
 186     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 187
 188     return if -f $pvessl_key_fn;
 189
 190     eval {
 191         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 192     };
 193
 194     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 195 }
 196
 197 sub gen_pve_www_key {
 198
 199     return if -f $pvewww_key_fn;
 200
 201     eval {
 202         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 203     };
 204
 205     die "unable to generate pve www key:\n$@" if $@;
 206 }
 207
 208 sub update_serial {
 209     my ($serial) = @_;
 210
 211     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 212 }
 213
 214 sub gen_pve_ssl_cert {
 215     my ($force, $nodename, $ip) = @_;
 216
 217     die "no node name specified" if !$nodename;
 218     die "no IP specified" if !$ip;
 219
 220     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 221
 222     return if !$force && -f $pvessl_cert_fn;
 223
 224     my $names = "IP:127.0.0.1,DNS:localhost";
 225
 226     my $rc = PVE::INotify::read_file('resolvconf');
 227
 228     $names .= ",IP:$ip";
 229
 230     my $fqdn = $nodename;
 231
 232     $names .= ",DNS:$nodename";
 233
 234     if ($rc && $rc->{search}) {
 235         $fqdn = $nodename . "." . $rc->{search};
 236         $names .= ",DNS:$fqdn";
 237     }
 238
 239     my $sslconf = <<__EOD;
 240 RANDFILE = /root/.rnd
 241 extensions = v3_req
 242
 243 [ req ]
 244 default_bits = 2048
 245 distinguished_name = req_distinguished_name
 246 req_extensions = v3_req
 247 prompt = no
 248 string_mask = nombstr
 249
 250 [ req_distinguished_name ]
 251 organizationalUnitName = PVE Cluster Node
 252 organizationName = Proxmox Virtual Environment
 253 commonName = $fqdn
 254
 255 [ v3_req ]
 256 basicConstraints = CA:FALSE
 257 nsCertType = server
 258 keyUsage = nonRepudiation, digitalSignature, keyEncipherment
 259 subjectAltName = $names
 260 __EOD
 261
 262     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 263     my $fh = IO::File->new ($cfgfn, "w");
 264     print $fh $sslconf;
 265     close ($fh);
 266
 267     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 268     unlink $reqfn;
 269
 270     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 271     eval {
 272         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 273                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 274     };
 275
 276     if (my $err = $@) {
 277         unlink $reqfn;
 278         unlink $cfgfn;
 279         die "unable to generate pve certificate request:\n$err";
 280     }
 281
 282     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 283
 284     eval {
 285         run_silent_cmd(['openssl', 'x509', '-req', '-in', $reqfn, '-days', '3650',
 286                         '-out', $pvessl_cert_fn, '-CAkey', $pveca_key_fn,
 287                         '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn,
 288                         '-extfile', $cfgfn]);
 289     };
 290
 291     if (my $err = $@) {
 292         unlink $reqfn;
 293         unlink $cfgfn;
 294         die "unable to generate pve ssl certificate:\n$err";
 295     }
 296
 297     unlink $cfgfn;
 298     unlink $reqfn;
 299 }
 300
 301 sub gen_pve_node_files {
 302     my ($nodename, $ip, $opt_force) = @_;
 303
 304     gen_local_dirs($nodename);
 305
 306     gen_auth_key();
 307
 308     # make sure we have a (cluster wide) secret
 309     # for CSRFR prevention
 310     gen_pve_www_key();
 311
 312     # make sure we have a (per node) private key
 313     gen_pve_ssl_key($nodename);
 314
 315     # make sure we have a CA
 316     my $force = gen_pveca_cert();
 317
 318     $force = 1 if $opt_force;
 319
 320     gen_pve_ssl_cert($force, $nodename, $ip);
 321 }
 322
 323 my $vzdump_cron_dummy = <<__EOD;
 324 # cluster wide vzdump cron schedule
 325 # Atomatically generated file - do not edit
 326
 327 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 328
 329 __EOD
 330
 331 sub gen_pve_vzdump_symlink {
 332
 333     my $filename = "/etc/pve/vzdump";
 334
 335     my $link_fn = "/etc/cron.d/vzdump";
 336
 337     if ((-f $filename) && (! -l $link_fn)) {
 338         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 339         symlink($filename, $link_fn);
 340     }
 341 }
 342
 343 sub gen_pve_vzdump_files {
 344
 345     my $filename = "/etc/pve/vzdump";
 346
 347     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 348         if ! -f $filename;
 349
 350     gen_pve_vzdump_symlink();
 351 };
 352
 353 my $versions = {};
 354 my $vmlist = {};
 355 my $clinfo = {};
 356
 357 my $ipcc_send_rec = sub {
 358     my ($msgid, $data) = @_;
 359
 360     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 361
 362     die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
 363
 364     return $res;
 365 };
 366
 367 my $ipcc_send_rec_json = sub {
 368     my ($msgid, $data) = @_;
 369
 370     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 371
 372     die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
 373
 374     return decode_json($res);
 375 };
 376
 377 my $ipcc_get_config = sub {
 378     my ($path) = @_;
 379
 380     my $bindata = pack "Z*", $path;
 381     my $res = PVE::IPCC::ipcc_send_rec(6, $bindata);
 382     if (!defined($res)) {
 383         return undef if ($! != 0);
 384         return '';
 385     }
 386
 387     return $res;
 388 };
 389
 390 my $ipcc_get_status = sub {
 391     my ($name, $nodename) = @_;
 392
 393     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 394     return PVE::IPCC::ipcc_send_rec(5, $bindata);
 395 };
 396
 397 my $ipcc_update_status = sub {
 398     my ($name, $data) = @_;
 399
 400     my $raw = ref($data) ? encode_json($data) : $data;
 401     # update status
 402     my $bindata = pack "Z[256]Z*", $name, $raw;
 403
 404     return &$ipcc_send_rec(4, $bindata);
 405 };
 406
 407 my $ipcc_log = sub {
 408     my ($priority, $ident, $tag, $msg) = @_;
 409
 410     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 411     bytes::length($tag) + 1, $ident, $tag, $msg;
 412
 413     return &$ipcc_send_rec(7, $bindata);
 414 };
 415
 416 my $ipcc_get_cluster_log = sub {
 417     my ($user, $max) = @_;
 418
 419     $max = 0 if !defined($max);
 420
 421     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 422     return &$ipcc_send_rec(8, $bindata);
 423 };
 424
 425 my $ccache = {};
 426
 427 sub cfs_update {
 428     eval {
 429         my $res = &$ipcc_send_rec_json(1);
 430         #warn "GOT1: " . Dumper($res);
 431         die "no starttime\n" if !$res->{starttime};
 432
 433         if (!$res->{starttime} || !$versions->{starttime} ||
 434             $res->{starttime} != $versions->{starttime}) {
 435             #print "detected changed starttime\n";
 436             $vmlist = {};
 437             $clinfo = {};
 438             $ccache = {};
 439         }
 440
 441         $versions = $res;
 442     };
 443     my $err = $@;
 444     if ($err) {
 445         $versions = {};
 446         $vmlist = {};
 447         $clinfo = {};
 448         $ccache = {};
 449         warn $err;
 450     }
 451
 452     eval {
 453         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 454             #warn "detected new clinfo\n";
 455             $clinfo = &$ipcc_send_rec_json(2);
 456         }
 457     };
 458     $err = $@;
 459     if ($err) {
 460         $clinfo = {};
 461         warn $err;
 462     }
 463
 464     eval {
 465         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 466             #warn "detected new vmlist1\n";
 467             $vmlist = &$ipcc_send_rec_json(3);
 468         }
 469     };
 470     $err = $@;
 471     if ($err) {
 472         $vmlist = {};
 473         warn $err;
 474     }
 475 }
 476
 477 sub get_vmlist {
 478     return $vmlist;
 479 }
 480
 481 sub get_clinfo {
 482     return $clinfo;
 483 }
 484
 485 sub get_members {
 486     return $clinfo->{nodelist};
 487 }
 488
 489 sub get_nodelist {
 490
 491     my $nodelist = $clinfo->{nodelist};
 492
 493     my $result = [];
 494
 495     my $nodename = PVE::INotify::nodename();
 496
 497     if (!$nodelist || !$nodelist->{$nodename}) {
 498         return [ $nodename ];
 499     }
 500
 501     return [ keys %$nodelist ];
 502 }
 503
 504 sub broadcast_tasklist {
 505     my ($data) = @_;
 506
 507     eval {
 508         &$ipcc_update_status("tasklist", $data);
 509     };
 510
 511     warn $@ if $@;
 512 }
 513
 514 my $tasklistcache = {};
 515
 516 sub get_tasklist {
 517     my ($nodename) = @_;
 518
 519     my $kvstore = $versions->{kvstore} || {};
 520
 521     my $nodelist = get_nodelist();
 522
 523     my $res = [];
 524     foreach my $node (@$nodelist) {
 525         next if $nodename && ($nodename ne $node);
 526         eval {
 527             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 528             my $cd = $tasklistcache->{$node};
 529             if (!$cd || !$ver || ($cd->{version} != $ver)) {
 530                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 531                 my $data = decode_json($raw);
 532                 push @$res, @$data;
 533                 $cd = $tasklistcache->{$node} = {
 534                     data => $data,
 535                     version => $ver,
 536                 };
 537             } elsif ($cd && $cd->{data}) {
 538                 push @$res, @{$cd->{data}};
 539             }
 540         };
 541         my $err = $@;
 542         syslog('err', $err) if $err;
 543     }
 544
 545     return $res;
 546 }
 547
 548 sub broadcast_rrd {
 549     my ($rrdid, $data) = @_;
 550
 551     eval {
 552         &$ipcc_update_status("rrd/$rrdid", $data);
 553     };
 554     my $err = $@;
 555
 556     warn $err if $err;
 557 }
 558
 559 my $last_rrd_dump = 0;
 560 my $last_rrd_data = "";
 561
 562 sub rrd_dump {
 563
 564     my $ctime = time();
 565
 566     my $diff = $ctime - $last_rrd_dump;
 567     if ($diff < 2) {
 568         return $last_rrd_data;
 569     }
 570
 571     my $raw;
 572     eval {
 573         $raw = &$ipcc_send_rec(10);
 574     };
 575     my $err = $@;
 576
 577     if ($err) {
 578         warn $err;
 579         return {};
 580     }
 581
 582     my $res = {};
 583
 584     while ($raw =~ s/^(.*)\n//) {
 585         my ($key, @ela) = split(/:/, $1);
 586         next if !$key;
 587         next if !(scalar(@ela) > 1);
 588         $res->{$key} = \@ela;
 589     }
 590
 591     $last_rrd_dump = $ctime;
 592     $last_rrd_data = $res;
 593
 594     return $res;
 595 }
 596
 597 sub create_rrd_data {
 598     my ($rrdname, $timeframe, $cf) = @_;
 599
 600     my $rrddir = "/var/lib/rrdcached/db";
 601
 602     my $rrd = "$rrddir/$rrdname";
 603
 604     my $setup = {
 605         hour =>  [ 60, 70 ],
 606         day  =>  [ 60*30, 70 ],
 607         week =>  [ 60*180, 70 ],
 608         month => [ 60*720, 70 ],
 609         year =>  [ 60*10080, 70 ],
 610     };
 611
 612     my ($reso, $count) = @{$setup->{$timeframe}};
 613     my $ctime  = $reso*int(time()/$reso);
 614     my $req_start = $ctime - $reso*$count;
 615
 616     $cf = "AVERAGE" if !$cf;
 617
 618     my @args = (
 619         "-s" => $req_start,
 620         "-e" => $ctime - 1,
 621         "-r" => $reso,
 622         );
 623
 624     my $socket = "/var/run/rrdcached.sock";
 625     push @args, "--daemon" => "unix:$socket" if -S $socket;
 626
 627     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 628
 629     my $err = RRDs::error;
 630     die "RRD error: $err\n" if $err;
 631
 632     die "got wrong time resolution ($step != $reso)\n"
 633         if $step != $reso;
 634
 635     my $res = [];
 636     my $fields = scalar(@$names);
 637     for my $line (@$data) {
 638         my $entry = { 'time' => $start };
 639         $start += $step;
 640         my $found_undefs;
 641         for (my $i = 0; $i < $fields; $i++) {
 642             my $name = $names->[$i];
 643             if (defined(my $val = $line->[$i])) {
 644                 $entry->{$name} = $val;
 645             } else {
 646                 # we only add entryies with all data defined
 647                 # extjs chart has problems with undefined values
 648                 $found_undefs = 1;
 649             }
 650         }
 651         push @$res, $entry if !$found_undefs;
 652     }
 653
 654     return $res;
 655 }
 656
 657 sub create_rrd_graph {
 658     my ($rrdname, $timeframe, $ds, $cf) = @_;
 659
 660     # Using RRD graph is clumsy - maybe it
 661     # is better to simply fetch the data, and do all display
 662     # related things with javascript (new extjs html5 graph library).
 663
 664     my $rrddir = "/var/lib/rrdcached/db";
 665
 666     my $rrd = "$rrddir/$rrdname";
 667
 668     my $filename = "$rrd.png";
 669
 670     my $setup = {
 671         hour =>  [ 60, 60 ],
 672         day  =>  [ 60*30, 70 ],
 673         week =>  [ 60*180, 70 ],
 674         month => [ 60*720, 70 ],
 675         year =>  [ 60*10080, 70 ],
 676     };
 677
 678     my ($reso, $count) = @{$setup->{$timeframe}};
 679
 680     my @args = (
 681         "--imgformat" => "PNG",
 682         "--border" => 0,
 683         "--height" => 200,
 684         "--width" => 800,
 685         "--start" => - $reso*$count,
 686         "--end" => 'now' ,
 687         );
 688
 689     my $socket = "/var/run/rrdcached.sock";
 690     push @args, "--daemon" => "unix:$socket" if -S $socket;
 691
 692     my @ids = PVE::Tools::split_list($ds);
 693
 694     my @coldef = ('#00ddff', '#ff0000');
 695
 696     $cf = "AVERAGE" if !$cf;
 697
 698     my $i = 0;
 699     foreach my $id (@ids) {
 700         my $col = $coldef[$i++] || die "fixme: no color definition";
 701         push @args, "DEF:${id}=$rrd:${id}:$cf";
 702         my $dataid = $id;
 703         if ($id eq 'cpu' || $id eq 'iowait') {
 704             push @args, "CDEF:${id}_per=${id},100,*";
 705             $dataid = "${id}_per";
 706         }
 707         push @args, "LINE2:${dataid}${col}:${id}";
 708     }
 709
 710     RRDs::graph($filename, @args);
 711
 712     my $err = RRDs::error;
 713     die "RRD error: $err\n" if $err;
 714
 715     return { filename => $filename };
 716 }
 717
 718 # a fast way to read files (avoid fuse overhead)
 719 sub get_config {
 720     my ($path) = @_;
 721
 722     return  &$ipcc_get_config($path);
 723 }
 724
 725 sub get_cluster_log {
 726     my ($user, $max) = @_;
 727
 728     return &$ipcc_get_cluster_log($user, $max);
 729 }
 730
 731 my $file_info = {};
 732
 733 sub cfs_register_file {
 734     my ($filename, $parser, $writer) = @_;
 735
 736     $observed->{$filename} || die "unknown file '$filename'";
 737
 738     die "file '$filename' already registered" if $file_info->{$filename};
 739
 740     $file_info->{$filename} = {
 741         parser => $parser,
 742         writer => $writer,
 743     };
 744 }
 745
 746 my $ccache_read = sub {
 747     my ($filename, $parser, $version) = @_;
 748
 749     $ccache->{$filename} = {} if !$ccache->{$filename};
 750
 751     my $ci = $ccache->{$filename};
 752
 753     if (!$ci->{version} || $ci->{version} != $version) {
 754
 755         my $data = get_config($filename);
 756         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 757         $ci->{version} = $version;
 758     }
 759
 760     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 761
 762     return $res;
 763 };
 764
 765 sub cfs_file_version {
 766     my ($filename) = @_;
 767
 768     my $version;
 769     my $infotag;
 770     if ($filename =~ m!^nodes/[^/]+/(openvz|qemu-server)/(\d+)\.conf$!) {
 771         my ($type, $vmid) = ($1, $2);
 772         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 773             $version = $vmlist->{ids}->{$vmid}->{version};
 774         }
 775         $infotag = "/$type/";
 776     } else {
 777         $infotag = $filename;
 778         $version = $versions->{$filename};
 779     }
 780
 781     my $info = $file_info->{$infotag} ||
 782         die "unknown file type '$filename'\n";
 783
 784     return wantarray ? ($version, $info) : $version;
 785 }
 786
 787 sub cfs_read_file {
 788     my ($filename) = @_;
 789
 790     my ($version, $info) = cfs_file_version($filename);
 791     my $parser = $info->{parser};
 792
 793     return &$ccache_read($filename, $parser, $version);
 794 }
 795
 796 sub cfs_write_file {
 797     my ($filename, $data) = @_;
 798
 799     my $info = $file_info->{$filename} || die "unknown file '$filename'";
 800
 801     my $writer = $info->{writer} || die "no writer defined";
 802
 803     my $fsname = "/etc/pve/$filename";
 804
 805     my $raw = &$writer($fsname, $data);
 806
 807     if (my $ci = $ccache->{$filename}) {
 808         $ci->{version} = undef;
 809     }
 810
 811     PVE::Tools::file_set_contents($fsname, $raw);
 812 }
 813
 814 my $cfs_lock = sub {
 815     my ($lockid, $timeout, $code, @param) = @_;
 816
 817     my $res;
 818
 819     # this timeout is for aquire the lock
 820     $timeout = 10 if !$timeout;
 821
 822     my $filename = "$lockdir/$lockid";
 823
 824     my $msg = "can't aquire cfs lock '$lockid'";
 825
 826     eval {
 827
 828         mkdir $lockdir;
 829
 830         if (! -d $lockdir) {
 831             die "$msg: pve cluster filesystem not online.\n";
 832         }
 833
 834         local $SIG{ALRM} = sub { die "got lock request timeout\n"; };
 835
 836         alarm ($timeout);
 837
 838         if (!(mkdir $filename)) {
 839             print STDERR "trying to aquire cfs lock '$lockid' ...";
 840             while (1) {
 841                 if (!(mkdir $filename)) {
 842                     (utime 0, 0, $filename); # cfs unlock request
 843                 } else {
 844                     print STDERR " OK\n";
 845                     last;
 846                 }
 847                 sleep(1);
 848             }
 849         }
 850
 851         # fixed command timeout: cfs locks have a timeout of 120
 852         # using 60 gives us another 60 seconds to abort the task
 853         alarm(60);
 854         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 855
 856         $res = &$code(@param);
 857
 858         alarm(0);
 859     };
 860
 861     my $err = $@;
 862
 863     alarm(0);
 864
 865     if ($err && ($err eq "got lock request timeout\n") &&
 866         !check_cfs_quorum()){
 867         $err = "$msg: no quorum!\n";
 868     }
 869
 870     if (!$err || $err !~ /^got lock timeout -/) {
 871         rmdir $filename; # cfs unlock
 872     }
 873
 874     if ($err) {
 875         $@ = $err;
 876         return undef;
 877     }
 878
 879     $@ = undef;
 880
 881     return $res;
 882 };
 883
 884 sub cfs_lock_file {
 885     my ($filename, $timeout, $code, @param) = @_;
 886
 887     my $info = $observed->{$filename} || die "unknown file '$filename'";
 888
 889     my $lockid = "file-$filename";
 890     $lockid =~ s/[.\/]/_/g;
 891
 892     &$cfs_lock($lockid, $timeout, $code, @param);
 893 }
 894
 895 sub cfs_lock_storage {
 896     my ($storeid, $timeout, $code, @param) = @_;
 897
 898     my $lockid = "storage-$storeid";
 899
 900     &$cfs_lock($lockid, $timeout, $code, @param);
 901 }
 902
 903 my $log_levels = {
 904     "emerg" => 0,
 905     "alert" => 1,
 906     "crit" => 2,
 907     "critical" => 2,
 908     "err" => 3,
 909     "error" => 3,
 910     "warn" => 4,
 911     "warning" => 4,
 912     "notice" => 5,
 913     "info" => 6,
 914     "debug" => 7,
 915 };
 916
 917 sub log_msg {
 918    my ($priority, $ident, $msg) = @_;
 919
 920    if (my $tmp = $log_levels->{$priority}) {
 921        $priority = $tmp;
 922    }
 923
 924    die "need numeric log priority" if $priority !~ /^\d+$/;
 925
 926    my $tag = PVE::SafeSyslog::tag();
 927
 928    $msg = "empty message" if !$msg;
 929
 930    $ident = "" if !$ident;
 931    $ident = encode("ascii", decode_utf8($ident),
 932                    sub { sprintf "\\u%04x", shift });
 933
 934    my $utf8 = decode_utf8($msg);
 935
 936    my $ascii = encode("ascii", $utf8, sub { sprintf "\\u%04x", shift });
 937
 938    if ($ident) {
 939        syslog($priority, "<%s> %s", $ident, $ascii);
 940    } else {
 941        syslog($priority, "%s", $ascii);
 942    }
 943
 944    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
 945
 946    syslog("err", "writing cluster log failed: $@") if $@;
 947 }
 948
 949 sub check_node_exists {
 950     my ($nodename, $noerr) = @_;
 951
 952     my $nodelist = $clinfo->{nodelist};
 953     return 1 if $nodelist && $nodelist->{$nodename};
 954
 955     return undef if $noerr;
 956
 957     die "no such cluster node '$nodename'\n";
 958 }
 959
 960 # this is also used to get the IP of the local node
 961 sub remote_node_ip {
 962     my ($nodename, $noerr) = @_;
 963
 964     my $nodelist = $clinfo->{nodelist};
 965     if ($nodelist && $nodelist->{$nodename}) {
 966         if (my $ip = $nodelist->{$nodename}->{ip}) {
 967             return $ip;
 968         }
 969     }
 970
 971     # fallback: try to get IP by other means
 972     my $packed_ip = gethostbyname($nodename);
 973     if (defined $packed_ip) {
 974         my $ip = inet_ntoa($packed_ip);
 975
 976         if ($ip =~ m/^127\./) {
 977             die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr;
 978             return undef;
 979         }
 980
 981         return $ip;
 982     }
 983
 984     die "unable to get IP for node '$nodename' - node offline?\n" if !$noerr;
 985
 986     return undef;
 987 }
 988
 989 # ssh related utility functions
 990
 991 sub ssh_merge_keys {
 992     # remove duplicate keys in $sshauthkeys
 993     # ssh-copy-id simply add keys, so the file can grow to large
 994
 995     my $data = '';
 996     if (-f $sshauthkeys) {
 997         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
 998         chomp($data);
 999     }
1000
1001     # always add ourself
1002     if (-f $ssh_rsa_id) {
1003         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
1004         chomp($pub);
1005         $data .= "\n$pub\n";
1006     }
1007
1008     my $newdata = "";
1009     my $vhash = {};
1010     while ($data && $data =~ s/^((.*?)(\n|$))//) {
1011         my $line = "$2\n";
1012         if ($line =~ m/^ssh-rsa\s+\S+\s+(\S+)$/) {
1013             $vhash->{$1} = $line;
1014         } else {
1015             $newdata .= $line;
1016         }
1017     }
1018
1019     $newdata .= join("", values(%$vhash));
1020
1021     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
1022 }
1023
1024 sub setup_ssh_keys {
1025
1026     # create ssh key if it does not exist
1027     if (! -f $ssh_rsa_id) {
1028         mkdir '/root/.ssh/';
1029         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1030     }
1031
1032     mkdir $authdir;
1033
1034     if (! -f $sshauthkeys) {
1035         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1036             close($fh);
1037         }
1038     }
1039
1040     warn "can't create shared ssh key database '$sshauthkeys'\n"
1041         if ! -f $sshauthkeys;
1042
1043     if (-f $rootsshauthkeys) {
1044         system("mv '$rootsshauthkeys' '$rootsshauthkeys.org'");
1045     }
1046
1047     if (! -l $rootsshauthkeys) {
1048         symlink $sshauthkeys, $rootsshauthkeys;
1049     }
1050     warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n"
1051         if ! -l $rootsshauthkeys;
1052
1053 }
1054
1055 sub ssh_unmerge_known_hosts {
1056     return if ! -l $sshglobalknownhosts;
1057
1058     my $old = '';
1059     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1060         if -f $sshknownhosts;
1061
1062     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1063 }
1064
1065 sub ssh_merge_known_hosts {
1066     my ($nodename, $ip_address, $createLink) = @_;
1067
1068     die "no node name specified" if !$nodename;
1069     die "no ip address specified" if !$ip_address;
1070
1071     mkdir $authdir;
1072
1073     if (! -f $sshknownhosts) {
1074         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1075             close($fh);
1076         }
1077     }
1078
1079     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1080
1081     my $new = '';
1082
1083     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1084         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1085     }
1086
1087     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1088     die "can't parse $ssh_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/;
1089     $hostkey = $1;
1090
1091     my $data = '';
1092     my $vhash = {};
1093
1094     my $found_nodename;
1095     my $found_local_ip;
1096
1097     my $merge_line = sub {
1098         my ($line, $all) = @_;
1099
1100         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1101             my $key = $1;
1102             my $rsakey = $2;
1103             if (!$vhash->{$key}) {
1104                 $vhash->{$key} = 1;
1105                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1106                     my $salt = decode_base64($1);
1107                     my $digest = $2;
1108                     my $hmac = Digest::HMAC_SHA1->new($salt);
1109                     $hmac->add($nodename);
1110                     my $hd = $hmac->b64digest . '=';
1111                     if ($digest eq $hd) {
1112                         if ($rsakey eq $hostkey) {
1113                             $found_nodename = 1;
1114                             $data .= $line;
1115                         }
1116                         return;
1117                     }
1118                     $hmac = Digest::HMAC_SHA1->new($salt);
1119                     $hmac->add($ip_address);
1120                     $hd = $hmac->b64digest . '=';
1121                     if ($digest eq $hd) {
1122                         if ($rsakey eq $hostkey) {
1123                             $found_local_ip = 1;
1124                             $data .= $line;
1125                         }
1126                         return;
1127                     }
1128                 }
1129                 $data .= $line;
1130             }
1131         } elsif ($all) {
1132             $data .= $line;
1133         }
1134     };
1135
1136     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1137         my $line = "$2\n";
1138         next if $line =~ m/^\s*$/; # skip empty lines
1139         next if $line =~ m/^#/; # skip comments
1140         &$merge_line($line, 1);
1141     }
1142
1143     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1144         my $line = "$2\n";
1145         next if $line =~ m/^\s*$/; # skip empty lines
1146         next if $line =~ m/^#/; # skip comments
1147         &$merge_line($line);
1148     }
1149
1150     my $addIndex = $$;
1151     my $add_known_hosts_entry  = sub {
1152         my ($name, $hostkey) = @_;
1153         $addIndex++;
1154         my $hmac = Digest::HMAC_SHA1->new("$addIndex" . time());
1155         my $b64salt = $hmac->b64digest . '=';
1156         $hmac = Digest::HMAC_SHA1->new(decode_base64($b64salt));
1157         $hmac->add($name);
1158         my $digest = $hmac->b64digest . '=';
1159         $data .= "|1|$b64salt|$digest $hostkey\n";
1160     };
1161
1162     if (!$found_nodename || !$found_local_ip) {
1163         &$add_known_hosts_entry($nodename, $hostkey) if !$found_nodename;
1164         &$add_known_hosts_entry($ip_address, $hostkey) if !$found_local_ip;
1165     }
1166
1167     PVE::Tools::file_set_contents($sshknownhosts, $data);
1168
1169     return if !$createLink;
1170
1171     unlink $sshglobalknownhosts;
1172     symlink $sshknownhosts, $sshglobalknownhosts;
1173
1174     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1175         if ! -l $sshglobalknownhosts;
1176
1177 }
1178
1179 my $keymaphash = PVE::Tools::kvmkeymaps();
1180 my $datacenter_schema = {
1181     type => "object",
1182     additionalProperties => 0,
1183     properties => {
1184         keyboard => {
1185             optional => 1,
1186             type => 'string',
1187             description => "Default keybord layout for vnc server.",
1188             enum => [ keys %$keymaphash ],
1189         },
1190         language => {
1191             optional => 1,
1192             type => 'string',
1193             description => "Default GUI language.",
1194             enum => [ 'en', 'de' ],
1195         },
1196         http_proxy => {
1197             optional => 1,
1198             type => 'string',
1199             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1200             pattern => "http://.*",
1201         },
1202     },
1203 };
1204
1205 # make schema accessible from outside (for documentation)
1206 sub get_datacenter_schema { return $datacenter_schema };
1207
1208 sub parse_datacenter_config {
1209     my ($filename, $raw) = @_;
1210
1211     return PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw);
1212 }
1213
1214 sub write_datacenter_config {
1215     my ($filename, $cfg) = @_;
1216
1217     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1218 }
1219
1220 cfs_register_file('datacenter.cfg',
1221                   \&parse_datacenter_config,
1222                   \&write_datacenter_config);