data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EEXIST ENOENT);
   6 use File::stat qw();
   7 use Socket;
   8 use Storable qw(dclone);
   9 use IO::File;
  10 use MIME::Base64;
  11 use Digest::SHA;
  12 use Digest::HMAC_SHA1;
  13 use Net::SSLeay;
  14 use PVE::Tools qw(run_command);
  15 use PVE::INotify;
  16 use PVE::IPCC;
  17 use PVE::SafeSyslog;
  18 use PVE::JSONSchema;
  19 use PVE::Network;
  20 use PVE::Cluster::IPCConst;
  21 use JSON;
  22 use RRDs;
  23 use Encode;
  24 use UUID;
  25 use base 'Exporter';
  26
  27 our @EXPORT_OK = qw(
  28 cfs_read_file
  29 cfs_write_file
  30 cfs_register_file
  31 cfs_lock_file);
  32
  33 use Data::Dumper; # fixme: remove
  34
  35 # x509 certificate utils
  36
  37 my $basedir = "/etc/pve";
  38 my $authdir = "$basedir/priv";
  39 my $lockdir = "/etc/pve/priv/lock";
  40
  41 # cfs and corosync files
  42 my $dbfile = "/var/lib/pve-cluster/config.db";
  43 my $dbbackupdir = "/var/lib/pve-cluster/backup";
  44 my $localclusterdir = "/etc/corosync";
  45 my $localclusterconf = "$localclusterdir/corosync.conf";
  46 my $authfile = "$localclusterdir/authkey";
  47 my $clusterconf = "$basedir/corosync.conf";
  48
  49 my $authprivkeyfn = "$authdir/authkey.key";
  50 my $authpubkeyfn = "$basedir/authkey.pub";
  51 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  52 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  53 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  54 # this is just a secret accessable by the web browser
  55 # and is used for CSRF prevention
  56 my $pvewww_key_fn = "$basedir/pve-www.key";
  57
  58 # ssh related files
  59 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  60 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  61 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  62 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  63 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  64 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  65 my $sshd_config_fn = "/etc/ssh/sshd_config";
  66 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  67 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  68 my $rootsshconfig = "/root/.ssh/config";
  69
  70 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
  71 # observed files are the one we can get directly through IPCC, they are cached
  72 # using a computed version and only those can be used by the cfs_*_file methods
  73 my $observed = {
  74     'vzdump.cron' => 1,
  75     'storage.cfg' => 1,
  76     'datacenter.cfg' => 1,
  77     'replication.cfg' => 1,
  78     'corosync.conf' => 1,
  79     'corosync.conf.new' => 1,
  80     'user.cfg' => 1,
  81     'domains.cfg' => 1,
  82     'priv/shadow.cfg' => 1,
  83     '/qemu-server/' => 1,
  84     '/openvz/' => 1,
  85     '/lxc/' => 1,
  86     'ha/crm_commands' => 1,
  87     'ha/manager_status' => 1,
  88     'ha/resources.cfg' => 1,
  89     'ha/groups.cfg' => 1,
  90     'ha/fence.cfg' => 1,
  91     'status.cfg' => 1,
  92     'ceph.conf' => 1,
  93 };
  94
  95 # only write output if something fails
  96 sub run_silent_cmd {
  97     my ($cmd) = @_;
  98
  99     my $outbuf = '';
 100     my $record = sub { $outbuf .= shift . "\n"; };
 101
 102     eval { run_command($cmd, outfunc => $record, errfunc => $record) };
 103
 104     if (my $err = $@) {
 105         print STDERR $outbuf;
 106         die $err;
 107     }
 108 }
 109
 110 sub check_cfs_quorum {
 111     my ($noerr) = @_;
 112
 113     # note: -w filename always return 1 for root, so wee need
 114     # to use File::lstat here
 115     my $st = File::stat::lstat("$basedir/local");
 116     my $quorate = ($st && (($st->mode & 0200) != 0));
 117
 118     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 119
 120     return $quorate;
 121 }
 122
 123 sub check_cfs_is_mounted {
 124     my ($noerr) = @_;
 125
 126     my $res = -l "$basedir/local";
 127
 128     die "pve configuration filesystem not mounted\n"
 129         if !$res && !$noerr;
 130
 131     return $res;
 132 }
 133
 134 sub gen_local_dirs {
 135     my ($nodename) = @_;
 136
 137     check_cfs_is_mounted();
 138
 139     my @required_dirs = (
 140         "$basedir/priv",
 141         "$basedir/nodes",
 142         "$basedir/nodes/$nodename",
 143         "$basedir/nodes/$nodename/lxc",
 144         "$basedir/nodes/$nodename/qemu-server",
 145         "$basedir/nodes/$nodename/openvz",
 146         "$basedir/nodes/$nodename/priv");
 147
 148     foreach my $dir (@required_dirs) {
 149         if (! -d $dir) {
 150             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 151         }
 152     }
 153 }
 154
 155 sub gen_auth_key {
 156
 157     return if -f "$authprivkeyfn";
 158
 159     check_cfs_is_mounted();
 160
 161     mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
 162
 163     run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 164
 165     run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 166 }
 167
 168 sub gen_pveca_key {
 169
 170     return if -f $pveca_key_fn;
 171
 172     eval {
 173         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 174     };
 175
 176     die "unable to generate pve ca key:\n$@" if $@;
 177 }
 178
 179 sub gen_pveca_cert {
 180
 181     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 182         return 0;
 183     }
 184
 185     gen_pveca_key();
 186
 187     # we try to generate an unique 'subject' to avoid browser problems
 188     # (reused serial numbers, ..)
 189     my $uuid;
 190     UUID::generate($uuid);
 191     my $uuid_str;
 192     UUID::unparse($uuid, $uuid_str);
 193
 194     eval {
 195         # wrap openssl with faketime to prevent bug #904
 196         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 197                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 198                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 199                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 200     };
 201
 202     die "generating pve root certificate failed:\n$@" if $@;
 203
 204     return 1;
 205 }
 206
 207 sub gen_pve_ssl_key {
 208     my ($nodename) = @_;
 209
 210     die "no node name specified" if !$nodename;
 211
 212     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 213
 214     return if -f $pvessl_key_fn;
 215
 216     eval {
 217         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 218     };
 219
 220     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 221 }
 222
 223 sub gen_pve_www_key {
 224
 225     return if -f $pvewww_key_fn;
 226
 227     eval {
 228         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 229     };
 230
 231     die "unable to generate pve www key:\n$@" if $@;
 232 }
 233
 234 sub update_serial {
 235     my ($serial) = @_;
 236
 237     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 238 }
 239
 240 sub gen_pve_ssl_cert {
 241     my ($force, $nodename, $ip) = @_;
 242
 243     die "no node name specified" if !$nodename;
 244     die "no IP specified" if !$ip;
 245
 246     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 247
 248     return if !$force && -f $pvessl_cert_fn;
 249
 250     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 251
 252     my $rc = PVE::INotify::read_file('resolvconf');
 253
 254     $names .= ",IP:$ip";
 255
 256     my $fqdn = $nodename;
 257
 258     $names .= ",DNS:$nodename";
 259
 260     if ($rc && $rc->{search}) {
 261         $fqdn = $nodename . "." . $rc->{search};
 262         $names .= ",DNS:$fqdn";
 263     }
 264
 265     my $sslconf = <<__EOD;
 266 RANDFILE = /root/.rnd
 267 extensions = v3_req
 268
 269 [ req ]
 270 default_bits = 2048
 271 distinguished_name = req_distinguished_name
 272 req_extensions = v3_req
 273 prompt = no
 274 string_mask = nombstr
 275
 276 [ req_distinguished_name ]
 277 organizationalUnitName = PVE Cluster Node
 278 organizationName = Proxmox Virtual Environment
 279 commonName = $fqdn
 280
 281 [ v3_req ]
 282 basicConstraints = CA:FALSE
 283 extendedKeyUsage = serverAuth
 284 subjectAltName = $names
 285 __EOD
 286
 287     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 288     my $fh = IO::File->new ($cfgfn, "w");
 289     print $fh $sslconf;
 290     close ($fh);
 291
 292     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 293     unlink $reqfn;
 294
 295     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 296     eval {
 297         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 298                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 299     };
 300
 301     if (my $err = $@) {
 302         unlink $reqfn;
 303         unlink $cfgfn;
 304         die "unable to generate pve certificate request:\n$err";
 305     }
 306
 307     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 308
 309     eval {
 310         # wrap openssl with faketime to prevent bug #904
 311         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
 312                         '-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
 313                         '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
 314                         '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
 315     };
 316
 317     if (my $err = $@) {
 318         unlink $reqfn;
 319         unlink $cfgfn;
 320         die "unable to generate pve ssl certificate:\n$err";
 321     }
 322
 323     unlink $cfgfn;
 324     unlink $reqfn;
 325 }
 326
 327 sub gen_pve_node_files {
 328     my ($nodename, $ip, $opt_force) = @_;
 329
 330     gen_local_dirs($nodename);
 331
 332     gen_auth_key();
 333
 334     # make sure we have a (cluster wide) secret
 335     # for CSRFR prevention
 336     gen_pve_www_key();
 337
 338     # make sure we have a (per node) private key
 339     gen_pve_ssl_key($nodename);
 340
 341     # make sure we have a CA
 342     my $force = gen_pveca_cert();
 343
 344     $force = 1 if $opt_force;
 345
 346     gen_pve_ssl_cert($force, $nodename, $ip);
 347 }
 348
 349 my $vzdump_cron_dummy = <<__EOD;
 350 # cluster wide vzdump cron schedule
 351 # Atomatically generated file - do not edit
 352
 353 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 354
 355 __EOD
 356
 357 sub gen_pve_vzdump_symlink {
 358
 359     my $filename = "/etc/pve/vzdump.cron";
 360
 361     my $link_fn = "/etc/cron.d/vzdump";
 362
 363     if ((-f $filename) && (! -l $link_fn)) {
 364         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 365         symlink($filename, $link_fn);
 366     }
 367 }
 368
 369 sub gen_pve_vzdump_files {
 370
 371     my $filename = "/etc/pve/vzdump.cron";
 372
 373     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 374         if ! -f $filename;
 375
 376     gen_pve_vzdump_symlink();
 377 };
 378
 379 my $versions = {};
 380 my $vmlist = {};
 381 my $clinfo = {};
 382
 383 my $ipcc_send_rec = sub {
 384     my ($msgid, $data) = @_;
 385
 386     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 387
 388     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 389
 390     return $res;
 391 };
 392
 393 my $ipcc_send_rec_json = sub {
 394     my ($msgid, $data) = @_;
 395
 396     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 397
 398     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 399
 400     return decode_json($res);
 401 };
 402
 403 my $ipcc_get_config = sub {
 404     my ($path) = @_;
 405
 406     my $bindata = pack "Z*", $path;
 407     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_CONFIG, $bindata);
 408     if (!defined($res)) {
 409         if ($! != 0) {
 410             return undef if $! == ENOENT;
 411             die "$!\n";
 412         }
 413         return '';
 414     }
 415
 416     return $res;
 417 };
 418
 419 my $ipcc_get_status = sub {
 420     my ($name, $nodename) = @_;
 421
 422     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 423     return PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_STATUS, $bindata);
 424 };
 425
 426 my $ipcc_update_status = sub {
 427     my ($name, $data) = @_;
 428
 429     my $raw = ref($data) ? encode_json($data) : $data;
 430     # update status
 431     my $bindata = pack "Z[256]Z*", $name, $raw;
 432
 433     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 434 };
 435
 436 my $ipcc_log = sub {
 437     my ($priority, $ident, $tag, $msg) = @_;
 438
 439     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 440     bytes::length($tag) + 1, $ident, $tag, $msg;
 441
 442     return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG, $bindata);
 443 };
 444
 445 my $ipcc_get_cluster_log = sub {
 446     my ($user, $max) = @_;
 447
 448     $max = 0 if !defined($max);
 449
 450     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 451     return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG, $bindata);
 452 };
 453
 454 my $ccache = {};
 455
 456 sub cfs_update {
 457     my ($fail) = @_;
 458     eval {
 459         my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION);
 460         #warn "GOT1: " . Dumper($res);
 461         die "no starttime\n" if !$res->{starttime};
 462
 463         if (!$res->{starttime} || !$versions->{starttime} ||
 464             $res->{starttime} != $versions->{starttime}) {
 465             #print "detected changed starttime\n";
 466             $vmlist = {};
 467             $clinfo = {};
 468             $ccache = {};
 469         }
 470
 471         $versions = $res;
 472     };
 473     my $err = $@;
 474     if ($err) {
 475         $versions = {};
 476         $vmlist = {};
 477         $clinfo = {};
 478         $ccache = {};
 479         die $err if $fail;
 480         warn $err;
 481     }
 482
 483     eval {
 484         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 485             #warn "detected new clinfo\n";
 486             $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO);
 487         }
 488     };
 489     $err = $@;
 490     if ($err) {
 491         $clinfo = {};
 492         die $err if $fail;
 493         warn $err;
 494     }
 495
 496     eval {
 497         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 498             #warn "detected new vmlist1\n";
 499             $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST);
 500         }
 501     };
 502     $err = $@;
 503     if ($err) {
 504         $vmlist = {};
 505         die $err if $fail;
 506         warn $err;
 507     }
 508 }
 509
 510 sub get_vmlist {
 511     return $vmlist;
 512 }
 513
 514 sub get_clinfo {
 515     return $clinfo;
 516 }
 517
 518 sub get_members {
 519     return $clinfo->{nodelist};
 520 }
 521
 522 sub get_nodelist {
 523
 524     my $nodelist = $clinfo->{nodelist};
 525
 526     my $result = [];
 527
 528     my $nodename = PVE::INotify::nodename();
 529
 530     if (!$nodelist || !$nodelist->{$nodename}) {
 531         return [ $nodename ];
 532     }
 533
 534     return [ keys %$nodelist ];
 535 }
 536
 537 # $data must be a chronological descending ordered array of tasks
 538 sub broadcast_tasklist {
 539     my ($data) = @_;
 540
 541     # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
 542     # from pmxcfs) - drop older items until we satisfy this constraint
 543     my $size = length(encode_json($data));
 544     while ($size >= (32 * 1024)) {
 545         pop @$data;
 546         $size = length(encode_json($data));
 547     }
 548
 549     eval {
 550         &$ipcc_update_status("tasklist", $data);
 551     };
 552
 553     warn $@ if $@;
 554 }
 555
 556 my $tasklistcache = {};
 557
 558 sub get_tasklist {
 559     my ($nodename) = @_;
 560
 561     my $kvstore = $versions->{kvstore} || {};
 562
 563     my $nodelist = get_nodelist();
 564
 565     my $res = [];
 566     foreach my $node (@$nodelist) {
 567         next if $nodename && ($nodename ne $node);
 568         eval {
 569             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 570             my $cd = $tasklistcache->{$node};
 571             if (!$cd || !$ver || !$cd->{version} ||
 572                 ($cd->{version} != $ver)) {
 573                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 574                 my $data = decode_json($raw);
 575                 push @$res, @$data;
 576                 $cd = $tasklistcache->{$node} = {
 577                     data => $data,
 578                     version => $ver,
 579                 };
 580             } elsif ($cd && $cd->{data}) {
 581                 push @$res, @{$cd->{data}};
 582             }
 583         };
 584         my $err = $@;
 585         syslog('err', $err) if $err;
 586     }
 587
 588     return $res;
 589 }
 590
 591 sub broadcast_rrd {
 592     my ($rrdid, $data) = @_;
 593
 594     eval {
 595         &$ipcc_update_status("rrd/$rrdid", $data);
 596     };
 597     my $err = $@;
 598
 599     warn $err if $err;
 600 }
 601
 602 my $last_rrd_dump = 0;
 603 my $last_rrd_data = "";
 604
 605 sub rrd_dump {
 606
 607     my $ctime = time();
 608
 609     my $diff = $ctime - $last_rrd_dump;
 610     if ($diff < 2) {
 611         return $last_rrd_data;
 612     }
 613
 614     my $raw;
 615     eval {
 616         $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP);
 617     };
 618     my $err = $@;
 619
 620     if ($err) {
 621         warn $err;
 622         return {};
 623     }
 624
 625     my $res = {};
 626
 627     if ($raw) {
 628         while ($raw =~ s/^(.*)\n//) {
 629             my ($key, @ela) = split(/:/, $1);
 630             next if !$key;
 631             next if !(scalar(@ela) > 1);
 632             $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
 633         }
 634     }
 635
 636     $last_rrd_dump = $ctime;
 637     $last_rrd_data = $res;
 638
 639     return $res;
 640 }
 641
 642 sub create_rrd_data {
 643     my ($rrdname, $timeframe, $cf) = @_;
 644
 645     my $rrddir = "/var/lib/rrdcached/db";
 646
 647     my $rrd = "$rrddir/$rrdname";
 648
 649     my $setup = {
 650         hour =>  [ 60, 70 ],
 651         day  =>  [ 60*30, 70 ],
 652         week =>  [ 60*180, 70 ],
 653         month => [ 60*720, 70 ],
 654         year =>  [ 60*10080, 70 ],
 655     };
 656
 657     my ($reso, $count) = @{$setup->{$timeframe}};
 658     my $ctime  = $reso*int(time()/$reso);
 659     my $req_start = $ctime - $reso*$count;
 660
 661     $cf = "AVERAGE" if !$cf;
 662
 663     my @args = (
 664         "-s" => $req_start,
 665         "-e" => $ctime - 1,
 666         "-r" => $reso,
 667         );
 668
 669     my $socket = "/var/run/rrdcached.sock";
 670     push @args, "--daemon" => "unix:$socket" if -S $socket;
 671
 672     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 673
 674     my $err = RRDs::error;
 675     die "RRD error: $err\n" if $err;
 676
 677     die "got wrong time resolution ($step != $reso)\n"
 678         if $step != $reso;
 679
 680     my $res = [];
 681     my $fields = scalar(@$names);
 682     for my $line (@$data) {
 683         my $entry = { 'time' => $start };
 684         $start += $step;
 685         for (my $i = 0; $i < $fields; $i++) {
 686             my $name = $names->[$i];
 687             if (defined(my $val = $line->[$i])) {
 688                 $entry->{$name} = $val;
 689             } else {
 690                 # leave empty fields undefined
 691                 # maybe make this configurable?
 692             }
 693         }
 694         push @$res, $entry;
 695     }
 696
 697     return $res;
 698 }
 699
 700 sub create_rrd_graph {
 701     my ($rrdname, $timeframe, $ds, $cf) = @_;
 702
 703     # Using RRD graph is clumsy - maybe it
 704     # is better to simply fetch the data, and do all display
 705     # related things with javascript (new extjs html5 graph library).
 706
 707     my $rrddir = "/var/lib/rrdcached/db";
 708
 709     my $rrd = "$rrddir/$rrdname";
 710
 711     my @ids = PVE::Tools::split_list($ds);
 712
 713     my $ds_txt = join('_', @ids);
 714
 715     my $filename = "${rrd}_${ds_txt}.png";
 716
 717     my $setup = {
 718         hour =>  [ 60, 60 ],
 719         day  =>  [ 60*30, 70 ],
 720         week =>  [ 60*180, 70 ],
 721         month => [ 60*720, 70 ],
 722         year =>  [ 60*10080, 70 ],
 723     };
 724
 725     my ($reso, $count) = @{$setup->{$timeframe}};
 726
 727     my @args = (
 728         "--imgformat" => "PNG",
 729         "--border" => 0,
 730         "--height" => 200,
 731         "--width" => 800,
 732         "--start" => - $reso*$count,
 733         "--end" => 'now' ,
 734         "--lower-limit" => 0,
 735         );
 736
 737     my $socket = "/var/run/rrdcached.sock";
 738     push @args, "--daemon" => "unix:$socket" if -S $socket;
 739
 740     my @coldef = ('#00ddff', '#ff0000');
 741
 742     $cf = "AVERAGE" if !$cf;
 743
 744     my $i = 0;
 745     foreach my $id (@ids) {
 746         my $col = $coldef[$i++] || die "fixme: no color definition";
 747         push @args, "DEF:${id}=$rrd:${id}:$cf";
 748         my $dataid = $id;
 749         if ($id eq 'cpu' || $id eq 'iowait') {
 750             push @args, "CDEF:${id}_per=${id},100,*";
 751             $dataid = "${id}_per";
 752         }
 753         push @args, "LINE2:${dataid}${col}:${id}";
 754     }
 755
 756     push @args, '--full-size-mode';
 757
 758     # we do not really store data into the file
 759     my $res = RRDs::graphv('-', @args);
 760
 761     my $err = RRDs::error;
 762     die "RRD error: $err\n" if $err;
 763
 764     return { filename => $filename, image => $res->{image} };
 765 }
 766
 767 # a fast way to read files (avoid fuse overhead)
 768 sub get_config {
 769     my ($path) = @_;
 770
 771     return &$ipcc_get_config($path);
 772 }
 773
 774 sub get_cluster_log {
 775     my ($user, $max) = @_;
 776
 777     return &$ipcc_get_cluster_log($user, $max);
 778 }
 779
 780 my $file_info = {};
 781
 782 sub cfs_register_file {
 783     my ($filename, $parser, $writer) = @_;
 784
 785     $observed->{$filename} || die "unknown file '$filename'";
 786
 787     die "file '$filename' already registered" if $file_info->{$filename};
 788
 789     $file_info->{$filename} = {
 790         parser => $parser,
 791         writer => $writer,
 792     };
 793 }
 794
 795 my $ccache_read = sub {
 796     my ($filename, $parser, $version) = @_;
 797
 798     $ccache->{$filename} = {} if !$ccache->{$filename};
 799
 800     my $ci = $ccache->{$filename};
 801
 802     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 803         # we always call the parser, even when the file does not exists
 804         # (in that case $data is undef)
 805         my $data = get_config($filename);
 806         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 807         $ci->{version} = $version;
 808     }
 809
 810     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 811
 812     return $res;
 813 };
 814
 815 sub cfs_file_version {
 816     my ($filename) = @_;
 817
 818     my $version;
 819     my $infotag;
 820     if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
 821         my ($type, $vmid) = ($1, $2);
 822         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 823             $version = $vmlist->{ids}->{$vmid}->{version};
 824         }
 825         $infotag = "/$type/";
 826     } else {
 827         $infotag = $filename;
 828         $version = $versions->{$filename};
 829     }
 830
 831     my $info = $file_info->{$infotag} ||
 832         die "unknown file type '$filename'\n";
 833
 834     return wantarray ? ($version, $info) : $version;
 835 }
 836
 837 sub cfs_read_file {
 838     my ($filename) = @_;
 839
 840     my ($version, $info) = cfs_file_version($filename);
 841     my $parser = $info->{parser};
 842
 843     return &$ccache_read($filename, $parser, $version);
 844 }
 845
 846 sub cfs_write_file {
 847     my ($filename, $data) = @_;
 848
 849     my ($version, $info) = cfs_file_version($filename);
 850
 851     my $writer = $info->{writer} || die "no writer defined";
 852
 853     my $fsname = "/etc/pve/$filename";
 854
 855     my $raw = &$writer($fsname, $data);
 856
 857     if (my $ci = $ccache->{$filename}) {
 858         $ci->{version} = undef;
 859     }
 860
 861     PVE::Tools::file_set_contents($fsname, $raw);
 862 }
 863
 864 my $cfs_lock = sub {
 865     my ($lockid, $timeout, $code, @param) = @_;
 866
 867     my $prev_alarm = alarm(0); # suspend outer alarm early
 868
 869     my $res;
 870     my $got_lock = 0;
 871
 872     # this timeout is for acquire the lock
 873     $timeout = 10 if !$timeout;
 874
 875     my $filename = "$lockdir/$lockid";
 876
 877     eval {
 878
 879         mkdir $lockdir;
 880
 881         if (! -d $lockdir) {
 882             die "pve cluster filesystem not online.\n";
 883         }
 884
 885         my $timeout_err = sub { die "got lock request timeout\n"; };
 886         local $SIG{ALRM} = $timeout_err;
 887
 888         while (1) {
 889             alarm ($timeout);
 890             $got_lock = mkdir($filename);
 891             $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
 892
 893             last if $got_lock;
 894
 895             $timeout_err->() if $timeout <= 0;
 896
 897             print STDERR "trying to acquire cfs lock '$lockid' ...\n";
 898             utime (0, 0, $filename); # cfs unlock request
 899             sleep(1);
 900         }
 901
 902         # fixed command timeout: cfs locks have a timeout of 120
 903         # using 60 gives us another 60 seconds to abort the task
 904         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 905         alarm(60);
 906
 907         cfs_update(); # make sure we read latest versions inside code()
 908
 909         $res = &$code(@param);
 910
 911         alarm(0);
 912     };
 913
 914     my $err = $@;
 915
 916     $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
 917
 918     rmdir $filename if $got_lock; # if we held the lock always unlock again
 919
 920     alarm($prev_alarm);
 921
 922     if ($err) {
 923         $@ = "error with cfs lock '$lockid': $err";
 924         return undef;
 925     }
 926
 927     $@ = undef;
 928
 929     return $res;
 930 };
 931
 932 sub cfs_lock_file {
 933     my ($filename, $timeout, $code, @param) = @_;
 934
 935     my $info = $observed->{$filename} || die "unknown file '$filename'";
 936
 937     my $lockid = "file-$filename";
 938     $lockid =~ s/[.\/]/_/g;
 939
 940     &$cfs_lock($lockid, $timeout, $code, @param);
 941 }
 942
 943 sub cfs_lock_storage {
 944     my ($storeid, $timeout, $code, @param) = @_;
 945
 946     my $lockid = "storage-$storeid";
 947
 948     &$cfs_lock($lockid, $timeout, $code, @param);
 949 }
 950
 951 sub cfs_lock_domain {
 952     my ($domainname, $timeout, $code, @param) = @_;
 953
 954     my $lockid = "domain-$domainname";
 955
 956     &$cfs_lock($lockid, $timeout, $code, @param);
 957 }
 958
 959 sub cfs_lock_acme {
 960     my ($account, $timeout, $code, @param) = @_;
 961
 962     my $lockid = "acme-$account";
 963
 964     &$cfs_lock($lockid, $timeout, $code, @param);
 965 }
 966
 967 my $log_levels = {
 968     "emerg" => 0,
 969     "alert" => 1,
 970     "crit" => 2,
 971     "critical" => 2,
 972     "err" => 3,
 973     "error" => 3,
 974     "warn" => 4,
 975     "warning" => 4,
 976     "notice" => 5,
 977     "info" => 6,
 978     "debug" => 7,
 979 };
 980
 981 sub log_msg {
 982    my ($priority, $ident, $msg) = @_;
 983
 984    if (my $tmp = $log_levels->{$priority}) {
 985        $priority = $tmp;
 986    }
 987
 988    die "need numeric log priority" if $priority !~ /^\d+$/;
 989
 990    my $tag = PVE::SafeSyslog::tag();
 991
 992    $msg = "empty message" if !$msg;
 993
 994    $ident = "" if !$ident;
 995    $ident = encode("ascii", $ident,
 996                    sub { sprintf "\\u%04x", shift });
 997
 998    my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
 999
1000    if ($ident) {
1001        syslog($priority, "<%s> %s", $ident, $ascii);
1002    } else {
1003        syslog($priority, "%s", $ascii);
1004    }
1005
1006    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
1007
1008    syslog("err", "writing cluster log failed: $@") if $@;
1009 }
1010
1011 sub check_vmid_unused {
1012     my ($vmid, $noerr) = @_;
1013
1014     my $vmlist = get_vmlist();
1015
1016     my $d = $vmlist->{ids}->{$vmid};
1017     return 1 if !defined($d);
1018
1019     return undef if $noerr;
1020
1021     my $vmtypestr =  $d->{type} eq 'qemu' ? 'VM' : 'CT';
1022     die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
1023 }
1024
1025 sub check_node_exists {
1026     my ($nodename, $noerr) = @_;
1027
1028     my $nodelist = $clinfo->{nodelist};
1029     return 1 if $nodelist && $nodelist->{$nodename};
1030
1031     return undef if $noerr;
1032
1033     die "no such cluster node '$nodename'\n";
1034 }
1035
1036 # this is also used to get the IP of the local node
1037 sub remote_node_ip {
1038     my ($nodename, $noerr) = @_;
1039
1040     my $nodelist = $clinfo->{nodelist};
1041     if ($nodelist && $nodelist->{$nodename}) {
1042         if (my $ip = $nodelist->{$nodename}->{ip}) {
1043             return $ip if !wantarray;
1044             my $family = $nodelist->{$nodename}->{address_family};
1045             if (!$family) {
1046                 $nodelist->{$nodename}->{address_family} =
1047                     $family =
1048                     PVE::Tools::get_host_address_family($ip);
1049             }
1050             return wantarray ? ($ip, $family) : $ip;
1051         }
1052     }
1053
1054     # fallback: try to get IP by other means
1055     return PVE::Network::get_ip_from_hostname($nodename, $noerr);
1056 }
1057
1058 sub get_local_migration_ip {
1059     my ($migration_network, $noerr) = @_;
1060
1061     my $cidr = $migration_network;
1062
1063     if (!defined($cidr)) {
1064         my $dc_conf = cfs_read_file('datacenter.cfg');
1065         $cidr = $dc_conf->{migration}->{network}
1066           if defined($dc_conf->{migration}->{network});
1067     }
1068
1069     if (defined($cidr)) {
1070         my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
1071
1072         die "could not get migration ip: no IP address configured on local " .
1073             "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
1074
1075         die "could not get migration ip: multiple IP address configured for " .
1076             "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
1077
1078         return @$ips[0];
1079     }
1080
1081     return undef;
1082 };
1083
1084 # ssh related utility functions
1085
1086 sub ssh_merge_keys {
1087     # remove duplicate keys in $sshauthkeys
1088     # ssh-copy-id simply add keys, so the file can grow to large
1089
1090     my $data = '';
1091     if (-f $sshauthkeys) {
1092         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
1093         chomp($data);
1094     }
1095
1096     my $found_backup;
1097     if (-f $rootsshauthkeysbackup) {
1098         $data .= "\n";
1099         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
1100         chomp($data);
1101         $found_backup = 1;
1102     }
1103
1104     # always add ourself
1105     if (-f $ssh_rsa_id) {
1106         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
1107         chomp($pub);
1108         $data .= "\n$pub\n";
1109     }
1110
1111     my $newdata = "";
1112     my $vhash = {};
1113     my @lines = split(/\n/, $data);
1114     foreach my $line (@lines) {
1115         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1116             next if $vhash->{$3}++;
1117         }
1118         $newdata .= "$line\n";
1119     }
1120
1121     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
1122
1123     if ($found_backup && -l $rootsshauthkeys) {
1124         # everything went well, so we can remove the backup
1125         unlink $rootsshauthkeysbackup;
1126     }
1127 }
1128
1129 sub setup_sshd_config {
1130     my () = @_;
1131
1132     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
1133
1134     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1135
1136     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1137         chomp $conf;
1138         $conf .= "\nPermitRootLogin yes\n";
1139     }
1140
1141     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
1142
1143     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
1144 }
1145
1146 sub setup_rootsshconfig {
1147
1148     # create ssh key if it does not exist
1149     if (! -f $ssh_rsa_id) {
1150         mkdir '/root/.ssh/';
1151         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1152     }
1153
1154     # create ssh config if it does not exist
1155     if (! -f $rootsshconfig) {
1156         mkdir '/root/.ssh';
1157         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
1158             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
1159             # changed order to put AES before Chacha20 (most hardware has AESNI)
1160             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
1161             close($fh);
1162         }
1163     }
1164 }
1165
1166 sub setup_ssh_keys {
1167
1168     mkdir $authdir;
1169
1170     my $import_ok;
1171
1172     if (! -f $sshauthkeys) {
1173         my $old;
1174         if (-f $rootsshauthkeys) {
1175             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
1176         }
1177         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1178             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
1179             close($fh);
1180             $import_ok = 1;
1181         }
1182     }
1183
1184     warn "can't create shared ssh key database '$sshauthkeys'\n"
1185         if ! -f $sshauthkeys;
1186
1187     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
1188         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1189             warn "rename $rootsshauthkeys failed - $!\n";
1190         }
1191     }
1192
1193     if (! -l $rootsshauthkeys) {
1194         symlink $sshauthkeys, $rootsshauthkeys;
1195     }
1196
1197     if (! -l $rootsshauthkeys) {
1198         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1199     } else {
1200         unlink $rootsshauthkeysbackup if $import_ok;
1201     }
1202 }
1203
1204 sub ssh_unmerge_known_hosts {
1205     return if ! -l $sshglobalknownhosts;
1206
1207     my $old = '';
1208     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1209         if -f $sshknownhosts;
1210
1211     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1212 }
1213
1214 sub ssh_merge_known_hosts {
1215     my ($nodename, $ip_address, $createLink) = @_;
1216
1217     die "no node name specified" if !$nodename;
1218     die "no ip address specified" if !$ip_address;
1219
1220     # ssh lowercases hostnames (aliases) before comparision, so we need too
1221     $nodename = lc($nodename);
1222     $ip_address = lc($ip_address);
1223
1224     mkdir $authdir;
1225
1226     if (! -f $sshknownhosts) {
1227         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1228             close($fh);
1229         }
1230     }
1231
1232     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1233
1234     my $new = '';
1235
1236     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1237         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1238     }
1239
1240     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1241     # Note: file sometimes containe emty lines at start, so we use multiline match
1242     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
1243     $hostkey = $1;
1244
1245     my $data = '';
1246     my $vhash = {};
1247
1248     my $found_nodename;
1249     my $found_local_ip;
1250
1251     my $merge_line = sub {
1252         my ($line, $all) = @_;
1253
1254         return if $line =~ m/^\s*$/; # skip empty lines
1255         return if $line =~ m/^#/; # skip comments
1256
1257         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1258             my $key = $1;
1259             my $rsakey = $2;
1260             if (!$vhash->{$key}) {
1261                 $vhash->{$key} = 1;
1262                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1263                     my $salt = decode_base64($1);
1264                     my $digest = $2;
1265                     my $hmac = Digest::HMAC_SHA1->new($salt);
1266                     $hmac->add($nodename);
1267                     my $hd = $hmac->b64digest . '=';
1268                     if ($digest eq $hd) {
1269                         if ($rsakey eq $hostkey) {
1270                             $found_nodename = 1;
1271                             $data .= $line;
1272                         }
1273                         return;
1274                     }
1275                     $hmac = Digest::HMAC_SHA1->new($salt);
1276                     $hmac->add($ip_address);
1277                     $hd = $hmac->b64digest . '=';
1278                     if ($digest eq $hd) {
1279                         if ($rsakey eq $hostkey) {
1280                             $found_local_ip = 1;
1281                             $data .= $line;
1282                         }
1283                         return;
1284                     }
1285                 } else {
1286                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
1287                     if ($key eq $ip_address) {
1288                         $found_local_ip = 1 if $rsakey eq $hostkey;
1289                     } elsif ($key eq $nodename) {
1290                         $found_nodename = 1 if $rsakey eq $hostkey;
1291                     }
1292                 }
1293                 $data .= $line;
1294             }
1295         } elsif ($all) {
1296             $data .= $line;
1297         }
1298     };
1299
1300     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1301         my $line = "$2\n";
1302         &$merge_line($line, 1);
1303     }
1304
1305     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1306         my $line = "$2\n";
1307         &$merge_line($line);
1308     }
1309
1310     # add our own key if not already there
1311     $data .= "$nodename $hostkey\n" if !$found_nodename;
1312     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
1313
1314     PVE::Tools::file_set_contents($sshknownhosts, $data);
1315
1316     return if !$createLink;
1317
1318     unlink $sshglobalknownhosts;
1319     symlink $sshknownhosts, $sshglobalknownhosts;
1320
1321     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1322         if ! -l $sshglobalknownhosts;
1323
1324 }
1325
1326 my $migration_format = {
1327     type => {
1328         default_key => 1,
1329         type => 'string',
1330         enum => ['secure', 'insecure'],
1331         description => "Migration traffic is encrypted using an SSH tunnel by " .
1332           "default. On secure, completely private networks this can be " .
1333           "disabled to increase performance.",
1334         default => 'secure',
1335     },
1336     network => {
1337         optional => 1,
1338         type => 'string', format => 'CIDR',
1339         format_description => 'CIDR',
1340         description => "CIDR of the (sub) network that is used for migration."
1341     },
1342 };
1343
1344 my $datacenter_schema = {
1345     type => "object",
1346     additionalProperties => 0,
1347     properties => {
1348         keyboard => {
1349             optional => 1,
1350             type => 'string',
1351             description => "Default keybord layout for vnc server.",
1352             enum => PVE::Tools::kvmkeymaplist(),
1353         },
1354         language => {
1355             optional => 1,
1356             type => 'string',
1357             description => "Default GUI language.",
1358             enum => [
1359                 'zh_CN',
1360                 'zh_TW',
1361                 'ca',
1362                 'en',
1363                 'eu',
1364                 'fr',
1365                 'de',
1366                 'it',
1367                 'es',
1368                 'ja',
1369                 'nb',
1370                 'nn',
1371                 'fa',
1372                 'pl',
1373                 'pt_BR',
1374                 'ru',
1375                 'sl',
1376                 'sv',
1377                 'tr',
1378             ],
1379         },
1380         http_proxy => {
1381             optional => 1,
1382             type => 'string',
1383             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1384             pattern => "http://.*",
1385         },
1386         migration_unsecure => {
1387             optional => 1,
1388             type => 'boolean',
1389             description => "Migration is secure using SSH tunnel by default. " .
1390               "For secure private networks you can disable it to speed up " .
1391               "migration. Deprecated, use the 'migration' property instead!",
1392         },
1393         migration => {
1394             optional => 1,
1395             type => 'string', format => $migration_format,
1396             description => "For cluster wide migration settings.",
1397         },
1398         console => {
1399             optional => 1,
1400             type => 'string',
1401             description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), an HTML5 based vnc viewer (noVNC), or an HTML5 based console client (xtermjs). If the selected viewer is not available (e.g. SPICE not activated for the VM), the fallback is noVNC.",
1402             enum => ['applet', 'vv', 'html5', 'xtermjs'],
1403         },
1404         email_from => {
1405             optional => 1,
1406             type => 'string',
1407             format => 'email-opt',
1408             description => "Specify email address to send notification from (default is root@\$hostname)",
1409         },
1410         max_workers => {
1411             optional => 1,
1412             type => 'integer',
1413             minimum => 1,
1414             description => "Defines how many workers (per node) are maximal started ".
1415               " on actions like 'stopall VMs' or task from the ha-manager.",
1416         },
1417         fencing => {
1418             optional => 1,
1419             type => 'string',
1420             default => 'watchdog',
1421             enum => [ 'watchdog', 'hardware', 'both' ],
1422             description => "Set the fencing mode of the HA cluster. Hardware mode " .
1423               "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
1424               " With both all two modes are used." .
1425               "\n\nWARNING: 'hardware' and 'both' are EXPERIMENTAL & WIP",
1426         },
1427         mac_prefix => {
1428             optional => 1,
1429             type => 'string',
1430             pattern => qr/[a-f0-9]{2}(?::[a-f0-9]{2}){0,2}:?/i,
1431             description => 'Prefix for autogenerated MAC addresses.',
1432         },
1433         bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
1434     },
1435 };
1436
1437 # make schema accessible from outside (for documentation)
1438 sub get_datacenter_schema { return $datacenter_schema };
1439
1440 sub parse_datacenter_config {
1441     my ($filename, $raw) = @_;
1442
1443     my $res = PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
1444
1445     if (my $migration = $res->{migration}) {
1446         $res->{migration} = PVE::JSONSchema::parse_property_string($migration_format, $migration);
1447     }
1448
1449     # for backwards compatibility only, new migration property has precedence
1450     if (defined($res->{migration_unsecure})) {
1451         if (defined($res->{migration}->{type})) {
1452             warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
1453               "set at same time! Ignore 'migration_unsecure'\n";
1454         } else {
1455             $res->{migration}->{type} = ($res->{migration_unsecure}) ? 'insecure' : 'secure';
1456         }
1457     }
1458
1459     # for backwards compatibility only, applet maps to html5
1460     if (defined($res->{console}) && $res->{console} eq 'applet') {
1461         $res->{console} = 'html5';
1462     }
1463
1464     return $res;
1465 }
1466
1467 sub write_datacenter_config {
1468     my ($filename, $cfg) = @_;
1469
1470     # map deprecated setting to new one
1471     if (defined($cfg->{migration_unsecure}) && !defined($cfg->{migration})) {
1472         my $migration_unsecure = delete $cfg->{migration_unsecure};
1473         $cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
1474     }
1475
1476     # map deprecated applet setting to html5
1477     if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
1478         $cfg->{console} = 'html5';
1479     }
1480
1481     if (my $migration = $cfg->{migration}) {
1482         $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
1483     }
1484
1485     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1486 }
1487
1488 cfs_register_file('datacenter.cfg',
1489                   \&parse_datacenter_config,
1490                   \&write_datacenter_config);
1491
1492 # X509 Certificate cache helper
1493
1494 my $cert_cache_nodes = {};
1495 my $cert_cache_timestamp = time();
1496 my $cert_cache_fingerprints = {};
1497
1498 sub update_cert_cache {
1499     my ($update_node, $clear) = @_;
1500
1501     syslog('info', "Clearing outdated entries from certificate cache")
1502         if $clear;
1503
1504     $cert_cache_timestamp = time() if !defined($update_node);
1505
1506     my $node_list = defined($update_node) ?
1507         [ $update_node ] : [ keys %$cert_cache_nodes ];
1508
1509     foreach my $node (@$node_list) {
1510         my $clear_old = sub {
1511             if (my $old_fp = $cert_cache_nodes->{$node}) {
1512                 # distrust old fingerprint
1513                 delete $cert_cache_fingerprints->{$old_fp};
1514                 # ensure reload on next proxied request
1515                 delete $cert_cache_nodes->{$node};
1516             }
1517         };
1518
1519         my $fp = eval { get_node_fingerprint($node) };
1520         if (my $err = $@) {
1521             warn "$err\n";
1522             &$clear_old() if $clear;
1523             next;
1524         }
1525
1526         my $old_fp = $cert_cache_nodes->{$node};
1527         $cert_cache_fingerprints->{$fp} = 1;
1528         $cert_cache_nodes->{$node} = $fp;
1529
1530         if (defined($old_fp) && $fp ne $old_fp) {
1531             delete $cert_cache_fingerprints->{$old_fp};
1532         }
1533     }
1534 }
1535
1536 # load and cache cert fingerprint once
1537 sub initialize_cert_cache {
1538     my ($node) = @_;
1539
1540     update_cert_cache($node)
1541         if defined($node) && !defined($cert_cache_nodes->{$node});
1542 }
1543
1544 sub read_ssl_cert_fingerprint {
1545     my ($cert_path) = @_;
1546
1547     my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
1548         or die "unable to read '$cert_path' - $!\n";
1549
1550     my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
1551     Net::SSLeay::BIO_free($bio);
1552
1553     die "unable to read certificate from '$cert_path'\n" if !$cert;
1554
1555     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1556     Net::SSLeay::X509_free($cert);
1557
1558     die "unable to get fingerprint for '$cert_path' - got empty value\n"
1559         if !defined($fp) || $fp eq '';
1560
1561     return $fp;
1562 }
1563
1564 sub get_node_fingerprint {
1565     my ($node) = @_;
1566
1567     my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
1568     my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
1569
1570     $cert_path = $custom_cert_path if -f $custom_cert_path;
1571
1572     return read_ssl_cert_fingerprint($cert_path);
1573 }
1574
1575
1576 sub check_cert_fingerprint {
1577     my ($cert) = @_;
1578
1579     # clear cache every 30 minutes at least
1580     update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
1581
1582     # get fingerprint of server certificate
1583     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1584     return 0 if !defined($fp) || $fp eq ''; # error
1585
1586     my $check = sub {
1587         for my $expected (keys %$cert_cache_fingerprints) {
1588             return 1 if $fp eq $expected;
1589         }
1590         return 0;
1591     };
1592
1593     return 1 if &$check();
1594
1595     # clear cache and retry at most once every minute
1596     if (time() - $cert_cache_timestamp >= 60) {
1597         syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
1598         update_cert_cache();
1599         return &$check();
1600     }
1601
1602     return 0;
1603 }
1604
1605 # bash completion helpers
1606
1607 sub complete_next_vmid {
1608
1609     my $vmlist = get_vmlist() || {};
1610     my $idlist = $vmlist->{ids} || {};
1611
1612     for (my $i = 100; $i < 10000; $i++) {
1613         return [$i] if !defined($idlist->{$i});
1614     }
1615
1616     return [];
1617 }
1618
1619 sub complete_vmid {
1620
1621     my $vmlist = get_vmlist();
1622     my $ids = $vmlist->{ids} || {};
1623
1624     return [ keys %$ids ];
1625 }
1626
1627 sub complete_local_vmid {
1628
1629     my $vmlist = get_vmlist();
1630     my $ids = $vmlist->{ids} || {};
1631
1632     my $nodename = PVE::INotify::nodename();
1633
1634     my $res = [];
1635     foreach my $vmid (keys %$ids) {
1636         my $d = $ids->{$vmid};
1637         next if !$d->{node} || $d->{node} ne $nodename;
1638         push @$res, $vmid;
1639     }
1640
1641     return $res;
1642 }
1643
1644 sub complete_migration_target {
1645
1646     my $res = [];
1647
1648     my $nodename = PVE::INotify::nodename();
1649
1650     my $nodelist = get_nodelist();
1651     foreach my $node (@$nodelist) {
1652         next if $node eq $nodename;
1653         push @$res, $node;
1654     }
1655
1656     return $res;
1657 }
1658
1659 sub get_ssh_info {
1660     my ($node, $network_cidr) = @_;
1661
1662     my $ip;
1663     if (defined($network_cidr)) {
1664         # Use mtunnel via to get the remote node's ip inside $network_cidr.
1665         # This goes over the regular network (iow. uses get_ssh_info() with
1666         # $network_cidr undefined.
1667         # FIXME: Use the REST API client for this after creating an API entry
1668         # for get_migration_ip.
1669         my $default_remote = get_ssh_info($node, undef);
1670         my $default_ssh = ssh_info_to_command($default_remote);
1671         my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
1672             '-migration_network', $network_cidr,
1673             '-get_migration_ip'
1674         ];
1675         PVE::Tools::run_command($cmd, outfunc => sub {
1676             my ($line) = @_;
1677             chomp $line;
1678             die "internal error: unexpected output from mtunnel\n"
1679                 if defined($ip);
1680             if ($line =~ /^ip: '(.*)'$/) {
1681                 $ip = $1;
1682             } else {
1683                 die "internal error: bad output from mtunnel\n"
1684                     if defined($ip);
1685             }
1686         });
1687         die "failed to get ip for node '$node' in network '$network_cidr'\n"
1688             if !defined($ip);
1689     } else {
1690         $ip = remote_node_ip($node);
1691     }
1692
1693     return {
1694         ip => $ip,
1695         name => $node,
1696         network => $network_cidr,
1697     };
1698 }
1699
1700 sub ssh_info_to_command_base {
1701     my ($info, @extra_options) = @_;
1702     return [
1703         '/usr/bin/ssh',
1704         '-e', 'none',
1705         '-o', 'BatchMode=yes',
1706         '-o', 'HostKeyAlias='.$info->{name},
1707         @extra_options
1708     ];
1709 }
1710
1711 sub ssh_info_to_command {
1712     my ($info, @extra_options) = @_;
1713     my $cmd = ssh_info_to_command_base($info, @extra_options);
1714     push @$cmd, "root\@$info->{ip}";
1715     return $cmd;
1716 }
1717
1718 sub assert_joinable {
1719     my ($ring0_addr, $ring1_addr, $force) = @_;
1720
1721     my $errors = '';
1722     my $error = sub { $errors .= "* $_[0]\n"; };
1723
1724     if (-f $authfile) {
1725         $error->("authentication key '$authfile' already exists");
1726     }
1727
1728     if (-f $clusterconf)  {
1729         $error->("cluster config '$clusterconf' already exists");
1730     }
1731
1732     my $vmlist = get_vmlist();
1733     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
1734         $error->("this host already contains virtual guests");
1735     }
1736
1737     if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
1738         $error->("corosync is already running, is this node already in a cluster?!");
1739     }
1740
1741     # check if corosync ring IPs are configured on the current nodes interfaces
1742     my $check_ip = sub {
1743         my $ip = shift // return;
1744         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
1745             my $host = $ip;
1746             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
1747             if ($@) {
1748                 $error->("cannot use '$host': $@\n") ;
1749                 return;
1750             }
1751         }
1752
1753         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
1754         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
1755
1756         $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n")
1757             if (scalar(@$configured_ips) != 1);
1758     };
1759
1760     $check_ip->($ring0_addr);
1761     $check_ip->($ring1_addr);
1762
1763     if ($errors) {
1764         warn "detected the following error(s):\n$errors";
1765         die "Check if node may join a cluster failed!\n" if !$force;
1766     }
1767 }
1768
1769 # NOTE: filesystem must be offline here, no DB changes allowed
1770 my $backup_cfs_database = sub {
1771     my ($dbfile) = @_;
1772
1773     mkdir $dbbackupdir;
1774
1775     my $ctime = time();
1776     my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
1777
1778     print "backup old database to '$backup_fn'\n";
1779
1780     my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \ ">${backup_fn}"] ];
1781     run_command($cmd, 'errmsg' => "cannot backup old database\n");
1782
1783     my $maxfiles = 10; # purge older backup
1784     my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql.gz> ];
1785
1786     if ((my $count = scalar(@$backups)) > $maxfiles) {
1787         foreach my $f (@$backups[$maxfiles..$count-1]) {
1788             next if $f !~ m/^(\S+)$/; # untaint
1789             print "delete old backup '$1'\n";
1790             unlink $1;
1791         }
1792     }
1793 };
1794
1795 sub join {
1796     my ($param) = @_;
1797
1798     my $nodename = PVE::INotify::nodename();
1799
1800     setup_sshd_config();
1801     setup_rootsshconfig();
1802     setup_ssh_keys();
1803
1804     # check if we can join with the given parameters and current node state
1805     my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
1806     assert_joinable($ring0_addr, $ring1_addr, $param->{force});
1807
1808     # make sure known_hosts is on local filesystem
1809     ssh_unmerge_known_hosts();
1810
1811     my $host = $param->{hostname};
1812     my $local_ip_address = remote_node_ip($nodename);
1813
1814     my $conn_args = {
1815         username => 'root@pam',
1816         password => $param->{password},
1817         cookie_name => 'PVEAuthCookie',
1818         protocol => 'https',
1819         host => $host,
1820         port => 8006,
1821     };
1822
1823     if (my $fp = $param->{fingerprint}) {
1824         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
1825     } else {
1826         # API schema ensures that we can only get here from CLI handler
1827         $conn_args->{manual_verification} = 1;
1828     }
1829
1830     print "Establishing API connection with host '$host'\n";
1831
1832     my $conn = PVE::APIClient::LWP->new(%$conn_args);
1833     $conn->login();
1834
1835     # login raises an exception on failure, so if we get here we're good
1836     print "Login succeeded.\n";
1837
1838     my $args = {};
1839     $args->{force} = $param->{force} if defined($param->{force});
1840     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
1841     $args->{votes} = $param->{votes} if defined($param->{votes});
1842     $args->{ring0_addr} = $ring0_addr // $local_ip_address;
1843     $args->{ring1_addr} = $ring1_addr if defined($ring1_addr);
1844
1845     print "Request addition of this node\n";
1846     my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
1847
1848     print "Join request OK, finishing setup locally\n";
1849
1850     # added successfuly - now prepare local node
1851     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
1852 }
1853
1854 sub finish_join {
1855     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
1856
1857     mkdir "$localclusterdir";
1858     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
1859     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
1860
1861     print "stopping pve-cluster service\n";
1862     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
1863     run_command($cmd, errmsg => "can't stop pve-cluster service");
1864
1865     $backup_cfs_database->($dbfile);
1866     unlink $dbfile;
1867
1868     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
1869     run_command($cmd, errmsg => "starting pve-cluster failed");
1870
1871     # wait for quorum
1872     my $printqmsg = 1;
1873     while (!check_cfs_quorum(1)) {
1874         if ($printqmsg) {
1875             print "waiting for quorum...";
1876             STDOUT->flush();
1877             $printqmsg = 0;
1878         }
1879         sleep(1);
1880     }
1881     print "OK\n" if !$printqmsg;
1882
1883     updatecerts_and_ssh(1);
1884
1885     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
1886     run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
1887
1888     print "successfully added node '$nodename' to cluster.\n";
1889 }
1890
1891 sub updatecerts_and_ssh {
1892     my ($force_new_cert, $silent) = @_;
1893
1894     my $p = sub { print "$_[0]\n" if !$silent };
1895
1896     setup_rootsshconfig();
1897
1898     gen_pve_vzdump_symlink();
1899
1900     if (!check_cfs_quorum(1)) {
1901         return undef if $silent;
1902         die "no quorum - unable to update files\n";
1903     }
1904
1905     setup_ssh_keys();
1906
1907     my $nodename = PVE::INotify::nodename();
1908     my $local_ip_address = remote_node_ip($nodename);
1909
1910     $p->("(re)generate node files");
1911     $p->("generate new node certificate") if $force_new_cert;
1912     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
1913
1914     $p->("merge authorized SSH keys and known hosts");
1915     ssh_merge_keys();
1916     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
1917     gen_pve_vzdump_files();
1918 }
1919
1920 1;