data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use POSIX;
   5 use File::stat qw();
   6 use Socket;
   7 use Storable qw(dclone);
   8 use IO::File;
   9 use MIME::Base64;
  10 use Digest::HMAC_SHA1;
  11 use PVE::Tools;
  12 use PVE::INotify;
  13 use PVE::IPCC;
  14 use PVE::SafeSyslog;
  15 use JSON;
  16 use RRDs;
  17 use Encode;
  18 use base 'Exporter';
  19
  20 our @EXPORT_OK = qw(
  21 cfs_read_file
  22 cfs_write_file
  23 cfs_register_file
  24 cfs_lock_file);
  25
  26 use Data::Dumper; # fixme: remove
  27
  28 # x509 certificate utils
  29
  30 my $basedir = "/etc/pve";
  31 my $authdir = "$basedir/priv";
  32 my $lockdir = "/etc/pve/priv/lock";
  33
  34 my $authprivkeyfn = "$authdir/authkey.key";
  35 my $authpubkeyfn = "$basedir/authkey.pub";
  36 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  37 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  38 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  39 # this is just a secret accessable by the web browser
  40 # and is used for CSRF prevention
  41 my $pvewww_key_fn = "$basedir/pve-www.key";
  42
  43 # ssh related files
  44 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  45 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  46 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  47 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  48 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  49 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  50 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  51
  52 my $observed = {
  53     'storage.cfg' => 1,
  54     'datacenter.cfg' => 1,
  55     'cluster.cfg' => 1,
  56     'user.cfg' => 1,
  57     'domains.cfg' => 1,
  58     'priv/shadow.cfg' => 1,
  59     '/qemu-server/' => 1,
  60     '/openvz/' => 1,
  61 };
  62
  63 # only write output if something fails
  64 sub run_silent_cmd {
  65     my ($cmd) = @_;
  66
  67     my $outbuf = '';
  68
  69     my $record_output = sub {
  70         $outbuf .= shift;
  71         $outbuf .= "\n";
  72     };
  73
  74     eval {
  75         PVE::Tools::run_command($cmd, outfunc => $record_output,
  76                                 errfunc => $record_output);
  77     };
  78
  79     my $err = $@;
  80
  81     if ($err) {
  82         print STDERR $outbuf;
  83         die $err;
  84     }
  85 }
  86
  87 sub check_cfs_quorum {
  88     my ($noerr) = @_;
  89
  90     # note: -w filename always return 1 for root, so wee need
  91     # to use File::lstat here
  92     my $st = File::stat::lstat("$basedir/local");
  93     my $quorate = ($st && (($st->mode & 0200) != 0));
  94
  95     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
  96
  97     return $quorate;
  98 }
  99
 100 sub check_cfs_is_mounted {
 101     my ($noerr) = @_;
 102
 103     my $res = -l "$basedir/local";
 104
 105     die "pve configuration filesystem not mounted\n"
 106         if !$res && !$noerr;
 107
 108     return $res;
 109 }
 110
 111 sub gen_local_dirs {
 112     my ($nodename) = @_;
 113
 114     check_cfs_is_mounted();
 115
 116     my @required_dirs = (
 117         "$basedir/priv",
 118         "$basedir/nodes",
 119         "$basedir/nodes/$nodename",
 120         "$basedir/nodes/$nodename/qemu-server",
 121         "$basedir/nodes/$nodename/openvz",
 122         "$basedir/nodes/$nodename/priv");
 123
 124     foreach my $dir (@required_dirs) {
 125         if (! -d $dir) {
 126             mkdir($dir) || die "unable to create directory '$dir' - $!\n";
 127         }
 128     }
 129 }
 130
 131 sub gen_auth_key {
 132
 133     return if -f "$authprivkeyfn";
 134
 135     check_cfs_is_mounted();
 136
 137     mkdir $authdir || die "unable to create dir '$authdir' - $!\n";
 138
 139     my $cmd = "openssl genrsa -out '$authprivkeyfn' 2048";
 140     run_silent_cmd($cmd);
 141
 142     $cmd = "openssl rsa -in '$authprivkeyfn' -pubout -out '$authpubkeyfn'";
 143     run_silent_cmd($cmd)
 144 }
 145
 146 sub gen_pveca_key {
 147
 148     return if -f $pveca_key_fn;
 149
 150     eval {
 151         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '2048']);
 152     };
 153
 154     die "unable to generate pve ca key:\n$@" if $@;
 155 }
 156
 157 sub gen_pveca_cert {
 158
 159     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 160         return 0;
 161     }
 162
 163     gen_pveca_key();
 164
 165     # we try to generate an unique 'subject' to avoid browser problems
 166     # (reused serial numbers, ..)
 167     my $nid = (split (/\s/, `md5sum '$pveca_key_fn'`))[0] || time();
 168
 169     eval {
 170         run_silent_cmd(['openssl', 'req', '-batch', '-days', '3650', '-new',
 171                         '-x509', '-nodes', '-key',
 172                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 173                         "/CN=Proxmox Virtual Environment/OU=$nid/O=PVE Cluster Manager CA/"]);
 174     };
 175
 176     die "generating pve root certificate failed:\n$@" if $@;
 177
 178     return 1;
 179 }
 180
 181 sub gen_pve_ssl_key {
 182     my ($nodename) = @_;
 183
 184     die "no node name specified" if !$nodename;
 185
 186     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 187
 188     return if -f $pvessl_key_fn;
 189
 190     eval {
 191         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 192     };
 193
 194     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 195 }
 196
 197 sub gen_pve_www_key {
 198
 199     return if -f $pvewww_key_fn;
 200
 201     eval {
 202         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 203     };
 204
 205     die "unable to generate pve www key:\n$@" if $@;
 206 }
 207
 208 sub update_serial {
 209     my ($serial) = @_;
 210
 211     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 212 }
 213
 214 sub gen_pve_ssl_cert {
 215     my ($force, $nodename, $ip) = @_;
 216
 217     die "no node name specified" if !$nodename;
 218     die "no IP specified" if !$ip;
 219
 220     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 221
 222     return if !$force && -f $pvessl_cert_fn;
 223
 224     my $names = "IP:127.0.0.1,DNS:localhost";
 225
 226     my $rc = PVE::INotify::read_file('resolvconf');
 227
 228     $names .= ",IP:$ip";
 229
 230     my $fqdn = $nodename;
 231
 232     $names .= ",DNS:$nodename";
 233
 234     if ($rc && $rc->{search}) {
 235         $fqdn = $nodename . "." . $rc->{search};
 236         $names .= ",DNS:$fqdn";
 237     }
 238
 239     my $sslconf = <<__EOD;
 240 RANDFILE = /root/.rnd
 241 extensions = v3_req
 242
 243 [ req ]
 244 default_bits = 2048
 245 distinguished_name = req_distinguished_name
 246 req_extensions = v3_req
 247 prompt = no
 248 string_mask = nombstr
 249
 250 [ req_distinguished_name ]
 251 organizationalUnitName = PVE Cluster Node
 252 organizationName = Proxmox Virtual Environment
 253 commonName = $fqdn
 254
 255 [ v3_req ]
 256 basicConstraints = CA:FALSE
 257 nsCertType = server
 258 keyUsage = nonRepudiation, digitalSignature, keyEncipherment
 259 subjectAltName = $names
 260 __EOD
 261
 262     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 263     my $fh = IO::File->new ($cfgfn, "w");
 264     print $fh $sslconf;
 265     close ($fh);
 266
 267     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 268     unlink $reqfn;
 269
 270     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 271     eval {
 272         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 273                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 274     };
 275
 276     if (my $err = $@) {
 277         unlink $reqfn;
 278         unlink $cfgfn;
 279         die "unable to generate pve certificate request:\n$err";
 280     }
 281
 282     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 283
 284     eval {
 285         run_silent_cmd(['openssl', 'x509', '-req', '-in', $reqfn, '-days', '3650',
 286                         '-out', $pvessl_cert_fn, '-CAkey', $pveca_key_fn,
 287                         '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn,
 288                         '-extfile', $cfgfn]);
 289     };
 290
 291     if (my $err = $@) {
 292         unlink $reqfn;
 293         unlink $cfgfn;
 294         die "unable to generate pve ssl certificate:\n$err";
 295     }
 296
 297     unlink $cfgfn;
 298     unlink $reqfn;
 299 }
 300
 301 sub gen_pve_node_files {
 302     my ($nodename, $ip, $opt_force) = @_;
 303
 304     gen_local_dirs($nodename);
 305
 306     gen_auth_key();
 307
 308     # make sure we have a (cluster wide) secret
 309     # for CSRFR prevention
 310     gen_pve_www_key();
 311
 312     # make sure we have a (per node) private key
 313     gen_pve_ssl_key($nodename);
 314
 315     # make sure we have a CA
 316     my $force = gen_pveca_cert();
 317
 318     $force = 1 if $opt_force;
 319
 320     gen_pve_ssl_cert($force, $nodename, $ip);
 321 }
 322
 323 my $versions = {};
 324 my $vmlist = {};
 325 my $clinfo = {};
 326
 327 my $ipcc_send_rec = sub {
 328     my ($msgid, $data) = @_;
 329
 330     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 331
 332     die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
 333
 334     return $res;
 335 };
 336
 337 my $ipcc_send_rec_json = sub {
 338     my ($msgid, $data) = @_;
 339
 340     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 341
 342     die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
 343
 344     return decode_json($res);
 345 };
 346
 347 my $ipcc_get_config = sub {
 348     my ($path) = @_;
 349
 350     my $bindata = pack "Z*", $path;
 351     my $res = PVE::IPCC::ipcc_send_rec(6, $bindata);
 352     if (!defined($res)) {
 353         return undef if ($! != 0);
 354         return '';
 355     }
 356
 357     return $res;
 358 };
 359
 360 my $ipcc_get_status = sub {
 361     my ($name, $nodename) = @_;
 362
 363     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 364     return PVE::IPCC::ipcc_send_rec(5, $bindata);
 365 };
 366
 367 my $ipcc_update_status = sub {
 368     my ($name, $data) = @_;
 369
 370     my $raw = ref($data) ? encode_json($data) : $data;
 371     # update status
 372     my $bindata = pack "Z[256]Z*", $name, $raw;
 373
 374     return &$ipcc_send_rec(4, $bindata);
 375 };
 376
 377 my $ipcc_log = sub {
 378     my ($priority, $ident, $tag, $msg) = @_;
 379
 380     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 381     bytes::length($tag) + 1, $ident, $tag, $msg;
 382
 383     return &$ipcc_send_rec(7, $bindata);
 384 };
 385
 386 my $ipcc_get_cluster_log = sub {
 387     my ($user, $max) = @_;
 388
 389     $max = 0 if !defined($max);
 390
 391     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 392     return &$ipcc_send_rec(8, $bindata);
 393 };
 394
 395 my $ccache = {};
 396
 397 sub cfs_update {
 398     eval {
 399         my $res = &$ipcc_send_rec_json(1);
 400         #warn "GOT1: " . Dumper($res);
 401         die "no starttime\n" if !$res->{starttime};
 402
 403         if (!$res->{starttime} || !$versions->{starttime} ||
 404             $res->{starttime} != $versions->{starttime}) {
 405             #print "detected changed starttime\n";
 406             $vmlist = {};
 407             $clinfo = {};
 408             $ccache = {};
 409         }
 410
 411         $versions = $res;
 412     };
 413     my $err = $@;
 414     if ($err) {
 415         $versions = {};
 416         $vmlist = {};
 417         $clinfo = {};
 418         $ccache = {};
 419         warn $err;
 420     }
 421
 422     eval {
 423         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 424             #warn "detected new clinfo\n";
 425             $clinfo = &$ipcc_send_rec_json(2);
 426         }
 427     };
 428     $err = $@;
 429     if ($err) {
 430         $clinfo = {};
 431         warn $err;
 432     }
 433
 434     eval {
 435         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 436             #warn "detected new vmlist1\n";
 437             $vmlist = &$ipcc_send_rec_json(3);
 438         }
 439     };
 440     $err = $@;
 441     if ($err) {
 442         $vmlist = {};
 443         warn $err;
 444     }
 445 }
 446
 447 sub get_vmlist {
 448     return $vmlist;
 449 }
 450
 451 sub get_clinfo {
 452     return $clinfo;
 453 }
 454
 455 sub get_members {
 456     return $clinfo->{nodelist};
 457 }
 458
 459 sub get_nodelist {
 460
 461     my $nodelist = $clinfo->{nodelist};
 462
 463     my $result = [];
 464
 465     my $nodename = PVE::INotify::nodename();
 466
 467     if (!$nodelist || !$nodelist->{$nodename}) {
 468         return [ $nodename ];
 469     }
 470
 471     return [ keys %$nodelist ];
 472 }
 473
 474 sub broadcast_tasklist {
 475     my ($data) = @_;
 476
 477     eval {
 478         &$ipcc_update_status("tasklist", $data);
 479     };
 480
 481     warn $@ if $@;
 482 }
 483
 484 my $tasklistcache = {};
 485
 486 sub get_tasklist {
 487     my ($nodename) = @_;
 488
 489     my $kvstore = $versions->{kvstore} || {};
 490
 491     my $nodelist = get_nodelist();
 492
 493     my $res = [];
 494     foreach my $node (@$nodelist) {
 495         next if $nodename && ($nodename ne $node);
 496         eval {
 497             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 498             my $cd = $tasklistcache->{$node};
 499             if (!$cd || !$ver || ($cd->{version} != $ver)) {
 500                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 501                 my $data = decode_json($raw);
 502                 push @$res, @$data;
 503                 $cd = $tasklistcache->{$node} = {
 504                     data => $data,
 505                     version => $ver,
 506                 };
 507             } elsif ($cd && $cd->{data}) {
 508                 push @$res, @{$cd->{data}};
 509             }
 510         };
 511         my $err = $@;
 512         syslog('err', $err) if $err;
 513     }
 514
 515     return $res;
 516 }
 517
 518 sub broadcast_rrd {
 519     my ($rrdid, $data) = @_;
 520
 521     eval {
 522         &$ipcc_update_status("rrd/$rrdid", $data);
 523     };
 524     my $err = $@;
 525
 526     warn $err if $err;
 527 }
 528
 529 my $last_rrd_dump = 0;
 530 my $last_rrd_data = "";
 531
 532 sub rrd_dump {
 533
 534     my $ctime = time();
 535
 536     my $diff = $ctime - $last_rrd_dump;
 537     if ($diff < 2) {
 538         return $last_rrd_data;
 539     }
 540
 541     my $raw;
 542     eval {
 543         $raw = &$ipcc_send_rec(10);
 544     };
 545     my $err = $@;
 546
 547     if ($err) {
 548         warn $err;
 549         return {};
 550     }
 551
 552     my $res = {};
 553
 554     while ($raw =~ s/^(.*)\n//) {
 555         my ($key, @ela) = split(/:/, $1);
 556         next if !$key;
 557         next if !(scalar(@ela) > 1);
 558         $res->{$key} = \@ela;
 559     }
 560
 561     $last_rrd_dump = $ctime;
 562     $last_rrd_data = $res;
 563
 564     return $res;
 565 }
 566
 567 sub create_rrd_data {
 568     my ($rrdname, $timeframe, $cf) = @_;
 569
 570     my $rrddir = "/var/lib/rrdcached/db";
 571
 572     my $rrd = "$rrddir/$rrdname";
 573
 574     my $setup = {
 575         hour =>  [ 60, 70 ],
 576         day  =>  [ 60*30, 70 ],
 577         week =>  [ 60*180, 70 ],
 578         month => [ 60*720, 70 ],
 579         year =>  [ 60*10080, 70 ],
 580     };
 581
 582     my ($reso, $count) = @{$setup->{$timeframe}};
 583     my $ctime  = $reso*int(time()/$reso);
 584     my $req_start = $ctime - $reso*$count;
 585
 586     $cf = "AVERAGE" if !$cf;
 587
 588     my @args = (
 589         "-s" => $req_start,
 590         "-e" => $ctime - 1,
 591         "-r" => $reso,
 592         );
 593
 594     my $socket = "/var/run/rrdcached.sock";
 595     push @args, "--daemon" => "unix:$socket" if -S $socket;
 596
 597     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 598
 599     my $err = RRDs::error;
 600     die "RRD error: $err\n" if $err;
 601
 602     die "got wrong time resolution ($step != $reso)\n"
 603         if $step != $reso;
 604
 605     my $res = [];
 606     my $fields = scalar(@$names);
 607     for my $line (@$data) {
 608         my $entry = { 'time' => $start };
 609         $start += $step;
 610         my $found_undefs;
 611         for (my $i = 0; $i < $fields; $i++) {
 612             my $name = $names->[$i];
 613             if (defined(my $val = $line->[$i])) {
 614                 $entry->{$name} = $val;
 615             } else {
 616                 # we only add entryies with all data defined
 617                 # extjs chart has problems with undefined values
 618                 $found_undefs = 1;
 619             }
 620         }
 621         push @$res, $entry if !$found_undefs;
 622     }
 623
 624     return $res;
 625 }
 626
 627 sub create_rrd_graph {
 628     my ($rrdname, $timeframe, $ds, $cf) = @_;
 629
 630     # Using RRD graph is clumsy - maybe it
 631     # is better to simply fetch the data, and do all display
 632     # related things with javascript (new extjs html5 graph library).
 633
 634     my $rrddir = "/var/lib/rrdcached/db";
 635
 636     my $rrd = "$rrddir/$rrdname";
 637
 638     my $filename = "$rrd.png";
 639
 640     my $setup = {
 641         hour =>  [ 60, 60 ],
 642         day  =>  [ 60*30, 70 ],
 643         week =>  [ 60*180, 70 ],
 644         month => [ 60*720, 70 ],
 645         year =>  [ 60*10080, 70 ],
 646     };
 647
 648     my ($reso, $count) = @{$setup->{$timeframe}};
 649
 650     my @args = (
 651         "--imgformat" => "PNG",
 652         "--border" => 0,
 653         "--height" => 200,
 654         "--width" => 800,
 655         "--start" => - $reso*$count,
 656         "--end" => 'now' ,
 657         );
 658
 659     my $socket = "/var/run/rrdcached.sock";
 660     push @args, "--daemon" => "unix:$socket" if -S $socket;
 661
 662     my @ids = PVE::Tools::split_list($ds);
 663
 664     my @coldef = ('#00ddff', '#ff0000');
 665
 666     $cf = "AVERAGE" if !$cf;
 667
 668     my $i = 0;
 669     foreach my $id (@ids) {
 670         my $col = $coldef[$i++] || die "fixme: no color definition";
 671         push @args, "DEF:${id}=$rrd:${id}:$cf";
 672         my $dataid = $id;
 673         if ($id eq 'cpu' || $id eq 'iowait') {
 674             push @args, "CDEF:${id}_per=${id},100,*";
 675             $dataid = "${id}_per";
 676         }
 677         push @args, "LINE2:${dataid}${col}:${id}";
 678     }
 679
 680     RRDs::graph($filename, @args);
 681
 682     my $err = RRDs::error;
 683     die "RRD error: $err\n" if $err;
 684
 685     return { filename => $filename };
 686 }
 687
 688 # a fast way to read files (avoid fuse overhead)
 689 sub get_config {
 690     my ($path) = @_;
 691
 692     return  &$ipcc_get_config($path);
 693 }
 694
 695 sub get_cluster_log {
 696     my ($user, $max) = @_;
 697
 698     return &$ipcc_get_cluster_log($user, $max);
 699 }
 700
 701 my $file_info = {};
 702
 703 sub cfs_register_file {
 704     my ($filename, $parser, $writer) = @_;
 705
 706     $observed->{$filename} || die "unknown file '$filename'";
 707
 708     die "file '$filename' already registered" if $file_info->{$filename};
 709
 710     $file_info->{$filename} = {
 711         parser => $parser,
 712         writer => $writer,
 713     };
 714 }
 715
 716 my $ccache_read = sub {
 717     my ($filename, $parser, $version) = @_;
 718
 719     $ccache->{$filename} = {} if !$ccache->{$filename};
 720
 721     my $ci = $ccache->{$filename};
 722
 723     if (!$ci->{version} || $ci->{version} != $version) {
 724
 725         my $data = get_config($filename);
 726         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 727         $ci->{version} = $version;
 728     }
 729
 730     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 731
 732     return $res;
 733 };
 734
 735 sub cfs_file_version {
 736     my ($filename) = @_;
 737
 738     my $version;
 739     my $infotag;
 740     if ($filename =~ m!^nodes/[^/]+/(openvz|qemu-server)/(\d+)\.conf$!) {
 741         my ($type, $vmid) = ($1, $2);
 742         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 743             $version = $vmlist->{ids}->{$vmid}->{version};
 744         }
 745         $infotag = "/$type/";
 746     } else {
 747         $infotag = $filename;
 748         $version = $versions->{$filename};
 749     }
 750
 751     my $info = $file_info->{$infotag} ||
 752         die "unknown file type '$filename'\n";
 753
 754     return wantarray ? ($version, $info) : $version;
 755 }
 756
 757 sub cfs_read_file {
 758     my ($filename) = @_;
 759
 760     my ($version, $info) = cfs_file_version($filename);
 761     my $parser = $info->{parser};
 762
 763     return &$ccache_read($filename, $parser, $version);
 764 }
 765
 766 sub cfs_write_file {
 767     my ($filename, $data) = @_;
 768
 769     my $info = $file_info->{$filename} || die "unknown file '$filename'";
 770
 771     my $writer = $info->{writer} || die "no writer defined";
 772
 773     my $fsname = "/etc/pve/$filename";
 774
 775     my $raw = &$writer($fsname, $data);
 776
 777     if (my $ci = $ccache->{$filename}) {
 778         $ci->{version} = undef;
 779     }
 780
 781     PVE::Tools::file_set_contents($fsname, $raw);
 782 }
 783
 784 my $cfs_lock = sub {
 785     my ($lockid, $timeout, $code, @param) = @_;
 786
 787     my $res;
 788
 789     # this timeout is for aquire the lock
 790     $timeout = 10 if !$timeout;
 791
 792     my $filename = "$lockdir/$lockid";
 793
 794     my $msg = "can't aquire cfs lock '$lockid'";
 795
 796     eval {
 797
 798         mkdir $lockdir;
 799
 800         if (! -d $lockdir) {
 801             die "$msg: pve cluster filesystem not online.\n";
 802         }
 803
 804         local $SIG{ALRM} = sub { die "got lock request timeout\n"; };
 805
 806         alarm ($timeout);
 807
 808         if (!(mkdir $filename)) {
 809             print STDERR "trying to aquire cfs lock '$lockid' ...";
 810             while (1) {
 811                 if (!(mkdir $filename)) {
 812                     (utime 0, 0, $filename); # cfs unlock request
 813                 } else {
 814                     print STDERR " OK\n";
 815                     last;
 816                 }
 817                 sleep(1);
 818             }
 819         }
 820
 821         # fixed command timeout: cfs locks have a timeout of 120
 822         # using 60 gives us another 60 seconds to abort the task
 823         alarm(60);
 824         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 825
 826         $res = &$code(@param);
 827
 828         alarm(0);
 829     };
 830
 831     my $err = $@;
 832
 833     alarm(0);
 834
 835     if ($err && ($err eq "got lock request timeout\n") &&
 836         !check_cfs_quorum()){
 837         $err = "$msg: no quorum!\n";
 838     }
 839
 840     if (!$err || $err !~ /^got lock timeout -/) {
 841         rmdir $filename; # cfs unlock
 842     }
 843
 844     if ($err) {
 845         $@ = $err;
 846         return undef;
 847     }
 848
 849     $@ = undef;
 850
 851     return $res;
 852 };
 853
 854 sub cfs_lock_file {
 855     my ($filename, $timeout, $code, @param) = @_;
 856
 857     my $info = $observed->{$filename} || die "unknown file '$filename'";
 858
 859     my $lockid = "file-$filename";
 860     $lockid =~ s/[.\/]/_/g;
 861
 862     &$cfs_lock($lockid, $timeout, $code, @param);
 863 }
 864
 865 sub cfs_lock_storage {
 866     my ($storeid, $timeout, $code, @param) = @_;
 867
 868     my $lockid = "storage-$storeid";
 869
 870     &$cfs_lock($lockid, $timeout, $code, @param);
 871 }
 872
 873 my $log_levels = {
 874     "emerg" => 0,
 875     "alert" => 1,
 876     "crit" => 2,
 877     "critical" => 2,
 878     "err" => 3,
 879     "error" => 3,
 880     "warn" => 4,
 881     "warning" => 4,
 882     "notice" => 5,
 883     "info" => 6,
 884     "debug" => 7,
 885 };
 886
 887 sub log_msg {
 888    my ($priority, $ident, $msg) = @_;
 889
 890    if (my $tmp = $log_levels->{$priority}) {
 891        $priority = $tmp;
 892    }
 893
 894    die "need numeric log priority" if $priority !~ /^\d+$/;
 895
 896    my $tag = PVE::SafeSyslog::tag();
 897
 898    $msg = "empty message" if !$msg;
 899
 900    $ident = "" if !$ident;
 901    $ident = encode("ascii", decode_utf8($ident),
 902                    sub { sprintf "\\u%04x", shift });
 903
 904    my $utf8 = decode_utf8($msg);
 905
 906    my $ascii = encode("ascii", $utf8, sub { sprintf "\\u%04x", shift });
 907
 908    if ($ident) {
 909        syslog($priority, "<%s> %s", $ident, $ascii);
 910    } else {
 911        syslog($priority, "%s", $ascii);
 912    }
 913
 914    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
 915
 916    syslog("err", "writing cluster log failed: $@") if $@;
 917 }
 918
 919 sub check_node_exists {
 920     my ($nodename, $noerr) = @_;
 921
 922     my $nodelist = $clinfo->{nodelist};
 923     return 1 if $nodelist && $nodelist->{$nodename};
 924
 925     return undef if $noerr;
 926
 927     die "no such cluster node '$nodename'\n";
 928 }
 929
 930 # this is also used to get the IP of the local node
 931 sub remote_node_ip {
 932     my ($nodename, $noerr) = @_;
 933
 934     my $nodelist = $clinfo->{nodelist};
 935     if ($nodelist && $nodelist->{$nodename}) {
 936         if (my $ip = $nodelist->{$nodename}->{ip}) {
 937             return $ip;
 938         }
 939     }
 940
 941     # fallback: try to get IP by other means
 942     my $packed_ip = gethostbyname($nodename);
 943     if (defined $packed_ip) {
 944         my $ip = inet_ntoa($packed_ip);
 945
 946         if ($ip =~ m/^127\./) {
 947             die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr;
 948             return undef;
 949         }
 950
 951         return $ip;
 952     }
 953
 954     die "unable to get IP for node '$nodename' - node offline?\n" if !$noerr;
 955
 956     return undef;
 957 }
 958
 959 # ssh related utility functions
 960
 961 sub ssh_merge_keys {
 962     # remove duplicate keys in $sshauthkeys
 963     # ssh-copy-id simply add keys, so the file can grow to large
 964
 965     my $data = '';
 966     if (-f $sshauthkeys) {
 967         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
 968         chomp($data);
 969     }
 970
 971     # always add ourself
 972     if (-f $ssh_rsa_id) {
 973         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
 974         chomp($pub);
 975         $data .= "\n$pub\n";
 976     }
 977
 978     my $newdata = "";
 979     my $vhash = {};
 980     while ($data && $data =~ s/^((.*?)(\n|$))//) {
 981         my $line = "$2\n";
 982         if ($line =~ m/^ssh-rsa\s+\S+\s+(\S+)$/) {
 983             $vhash->{$1} = $line;
 984         } else {
 985             $newdata .= $line;
 986         }
 987     }
 988
 989     $newdata .= join("", values(%$vhash));
 990
 991     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
 992 }
 993
 994 sub setup_ssh_keys {
 995
 996     # create ssh key if it does not exist
 997     if (! -f $ssh_rsa_id) {
 998         mkdir '/root/.ssh/';
 999         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1000     }
1001
1002     mkdir $authdir;
1003
1004     if (! -f $sshauthkeys) {
1005         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1006             close($fh);
1007         }
1008     }
1009
1010     warn "can't create shared ssh key database '$sshauthkeys'\n"
1011         if ! -f $sshauthkeys;
1012
1013     if (-f $rootsshauthkeys) {
1014         system("mv '$rootsshauthkeys' '$rootsshauthkeys.org'");
1015     }
1016
1017     if (! -l $rootsshauthkeys) {
1018         symlink $sshauthkeys, $rootsshauthkeys;
1019     }
1020     warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n"
1021         if ! -l $rootsshauthkeys;
1022
1023 }
1024
1025 sub ssh_unmerge_known_hosts {
1026     return if ! -l $sshglobalknownhosts;
1027
1028     my $old = '';
1029     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1030         if -f $sshknownhosts;
1031
1032     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1033 }
1034
1035 sub ssh_merge_known_hosts {
1036     my ($nodename, $ip_address, $createLink) = @_;
1037
1038     die "no node name specified" if !$nodename;
1039     die "no ip address specified" if !$ip_address;
1040
1041     mkdir $authdir;
1042
1043     if (! -f $sshknownhosts) {
1044         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1045             close($fh);
1046         }
1047     }
1048
1049     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1050
1051     my $new = '';
1052
1053     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1054         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1055     }
1056
1057     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1058     die "can't parse $ssh_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/;
1059     $hostkey = $1;
1060
1061     my $data = '';
1062     my $vhash = {};
1063
1064     my $found_nodename;
1065     my $found_local_ip;
1066
1067     my $merge_line = sub {
1068         my ($line, $all) = @_;
1069
1070         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1071             my $key = $1;
1072             my $rsakey = $2;
1073             if (!$vhash->{$key}) {
1074                 $vhash->{$key} = 1;
1075                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1076                     my $salt = decode_base64($1);
1077                     my $digest = $2;
1078                     my $hmac = Digest::HMAC_SHA1->new($salt);
1079                     $hmac->add($nodename);
1080                     my $hd = $hmac->b64digest . '=';
1081                     if ($digest eq $hd) {
1082                         if ($rsakey eq $hostkey) {
1083                             $found_nodename = 1;
1084                             $data .= $line;
1085                         }
1086                         return;
1087                     }
1088                     $hmac = Digest::HMAC_SHA1->new($salt);
1089                     $hmac->add($ip_address);
1090                     $hd = $hmac->b64digest . '=';
1091                     if ($digest eq $hd) {
1092                         if ($rsakey eq $hostkey) {
1093                             $found_local_ip = 1;
1094                             $data .= $line;
1095                         }
1096                         return;
1097                     }
1098                 }
1099                 $data .= $line;
1100             }
1101         } elsif ($all) {
1102             $data .= $line;
1103         }
1104     };
1105
1106     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1107         my $line = "$2\n";
1108         next if $line =~ m/^\s*$/; # skip empty lines
1109         next if $line =~ m/^#/; # skip comments
1110         &$merge_line($line, 1);
1111     }
1112
1113     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1114         my $line = "$2\n";
1115         next if $line =~ m/^\s*$/; # skip empty lines
1116         next if $line =~ m/^#/; # skip comments
1117         &$merge_line($line);
1118     }
1119
1120     my $addIndex = $$;
1121     my $add_known_hosts_entry  = sub {
1122         my ($name, $hostkey) = @_;
1123         $addIndex++;
1124         my $hmac = Digest::HMAC_SHA1->new("$addIndex" . time());
1125         my $b64salt = $hmac->b64digest . '=';
1126         $hmac = Digest::HMAC_SHA1->new(decode_base64($b64salt));
1127         $hmac->add($name);
1128         my $digest = $hmac->b64digest . '=';
1129         $data .= "|1|$b64salt|$digest $hostkey\n";
1130     };
1131
1132     if (!$found_nodename || !$found_local_ip) {
1133         &$add_known_hosts_entry($nodename, $hostkey) if !$found_nodename;
1134         &$add_known_hosts_entry($ip_address, $hostkey) if !$found_local_ip;
1135     }
1136
1137     PVE::Tools::file_set_contents($sshknownhosts, $data);
1138
1139     return if !$createLink;
1140
1141     unlink $sshglobalknownhosts;
1142     symlink $sshknownhosts, $sshglobalknownhosts;
1143
1144     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1145         if ! -l $sshglobalknownhosts;
1146
1147 }
1148
1149 my $keymaphash = PVE::Tools::kvmkeymaps();
1150 my $datacenter_schema = {
1151     type => "object",
1152     additionalProperties => 0,
1153     properties => {
1154         keyboard => {
1155             optional => 1,
1156             type => 'string',
1157             description => "Default keybord layout for vnc server.",
1158             enum => [ keys %$keymaphash ],
1159         },
1160         language => {
1161             optional => 1,
1162             type => 'string',
1163             description => "Default GUI language.",
1164             enum => [ 'en', 'de' ],
1165         },
1166         http_proxy => {
1167             optional => 1,
1168             type => 'string',
1169             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1170             pattern => "http://.*",
1171         },
1172     },
1173 };
1174
1175 # make schema accessible from outside (for documentation)
1176 sub get_datacenter_schema { return $datacenter_schema };
1177
1178 sub parse_datacenter_config {
1179     my ($filename, $raw) = @_;
1180
1181     return PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw);
1182 }
1183
1184 sub write_datacenter_config {
1185     my ($filename, $cfg) = @_;
1186
1187     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1188 }
1189
1190 cfs_register_file('datacenter.cfg',
1191                   \&parse_datacenter_config,
1192                   \&write_datacenter_config);