data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Digest::HMAC_SHA1;
   7 use Digest::SHA;
   8 use Encode;
   9 use File::stat qw();
  10 use IO::File;
  11 use JSON;
  12 use MIME::Base64;
  13 use Net::SSLeay;
  14 use POSIX qw(EEXIST ENOENT);
  15 use RRDs;
  16 use Socket;
  17 use Storable qw(dclone);
  18 use UUID;
  19
  20 use PVE::INotify;
  21 use PVE::IPCC;
  22 use PVE::JSONSchema;
  23 use PVE::Network;
  24 use PVE::SafeSyslog;
  25 use PVE::Tools qw(run_command);
  26
  27 use PVE::Cluster::IPCConst;
  28
  29 use base 'Exporter';
  30
  31 our @EXPORT_OK = qw(
  32 cfs_read_file
  33 cfs_write_file
  34 cfs_register_file
  35 cfs_lock_file);
  36
  37 use Data::Dumper; # fixme: remove
  38
  39 # x509 certificate utils
  40
  41 my $basedir = "/etc/pve";
  42 my $authdir = "$basedir/priv";
  43 my $lockdir = "/etc/pve/priv/lock";
  44
  45 # cfs and corosync files
  46 my $dbfile = "/var/lib/pve-cluster/config.db";
  47 my $dbbackupdir = "/var/lib/pve-cluster/backup";
  48 my $localclusterdir = "/etc/corosync";
  49 my $localclusterconf = "$localclusterdir/corosync.conf";
  50 my $authfile = "$localclusterdir/authkey";
  51 my $clusterconf = "$basedir/corosync.conf";
  52
  53 my $authprivkeyfn = "$authdir/authkey.key";
  54 my $authpubkeyfn = "$basedir/authkey.pub";
  55 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  56 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  57 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  58 # this is just a secret accessable by the web browser
  59 # and is used for CSRF prevention
  60 my $pvewww_key_fn = "$basedir/pve-www.key";
  61
  62 # ssh related files
  63 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  64 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  65 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  66 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  67 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  68 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  69 my $sshd_config_fn = "/etc/ssh/sshd_config";
  70 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  71 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  72 my $rootsshconfig = "/root/.ssh/config";
  73
  74 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
  75 # observed files are the one we can get directly through IPCC, they are cached
  76 # using a computed version and only those can be used by the cfs_*_file methods
  77 my $observed = {
  78     'vzdump.cron' => 1,
  79     'storage.cfg' => 1,
  80     'datacenter.cfg' => 1,
  81     'replication.cfg' => 1,
  82     'corosync.conf' => 1,
  83     'corosync.conf.new' => 1,
  84     'user.cfg' => 1,
  85     'domains.cfg' => 1,
  86     'priv/shadow.cfg' => 1,
  87     'priv/tfa.cfg' => 1,
  88     '/qemu-server/' => 1,
  89     '/openvz/' => 1,
  90     '/lxc/' => 1,
  91     'ha/crm_commands' => 1,
  92     'ha/manager_status' => 1,
  93     'ha/resources.cfg' => 1,
  94     'ha/groups.cfg' => 1,
  95     'ha/fence.cfg' => 1,
  96     'status.cfg' => 1,
  97     'ceph.conf' => 1,
  98 };
  99
 100 # only write output if something fails
 101 sub run_silent_cmd {
 102     my ($cmd) = @_;
 103
 104     my $outbuf = '';
 105     my $record = sub { $outbuf .= shift . "\n"; };
 106
 107     eval { run_command($cmd, outfunc => $record, errfunc => $record) };
 108
 109     if (my $err = $@) {
 110         print STDERR $outbuf;
 111         die $err;
 112     }
 113 }
 114
 115 sub check_cfs_quorum {
 116     my ($noerr) = @_;
 117
 118     # note: -w filename always return 1 for root, so wee need
 119     # to use File::lstat here
 120     my $st = File::stat::lstat("$basedir/local");
 121     my $quorate = ($st && (($st->mode & 0200) != 0));
 122
 123     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 124
 125     return $quorate;
 126 }
 127
 128 sub check_cfs_is_mounted {
 129     my ($noerr) = @_;
 130
 131     my $res = -l "$basedir/local";
 132
 133     die "pve configuration filesystem not mounted\n"
 134         if !$res && !$noerr;
 135
 136     return $res;
 137 }
 138
 139 sub gen_local_dirs {
 140     my ($nodename) = @_;
 141
 142     check_cfs_is_mounted();
 143
 144     my @required_dirs = (
 145         "$basedir/priv",
 146         "$basedir/nodes",
 147         "$basedir/nodes/$nodename",
 148         "$basedir/nodes/$nodename/lxc",
 149         "$basedir/nodes/$nodename/qemu-server",
 150         "$basedir/nodes/$nodename/openvz",
 151         "$basedir/nodes/$nodename/priv");
 152
 153     foreach my $dir (@required_dirs) {
 154         if (! -d $dir) {
 155             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 156         }
 157     }
 158 }
 159
 160 sub gen_auth_key {
 161
 162     return if -f "$authprivkeyfn";
 163
 164     check_cfs_is_mounted();
 165
 166     cfs_lock_authkey(undef, sub {
 167         mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
 168
 169         run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 170
 171         run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 172     });
 173
 174     die "$@\n" if $@;
 175 }
 176
 177 sub gen_pveca_key {
 178
 179     return if -f $pveca_key_fn;
 180
 181     eval {
 182         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 183     };
 184
 185     die "unable to generate pve ca key:\n$@" if $@;
 186 }
 187
 188 sub gen_pveca_cert {
 189
 190     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 191         return 0;
 192     }
 193
 194     gen_pveca_key();
 195
 196     # we try to generate an unique 'subject' to avoid browser problems
 197     # (reused serial numbers, ..)
 198     my $uuid;
 199     UUID::generate($uuid);
 200     my $uuid_str;
 201     UUID::unparse($uuid, $uuid_str);
 202
 203     eval {
 204         # wrap openssl with faketime to prevent bug #904
 205         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 206                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 207                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 208                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 209     };
 210
 211     die "generating pve root certificate failed:\n$@" if $@;
 212
 213     return 1;
 214 }
 215
 216 sub gen_pve_ssl_key {
 217     my ($nodename) = @_;
 218
 219     die "no node name specified" if !$nodename;
 220
 221     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 222
 223     return if -f $pvessl_key_fn;
 224
 225     eval {
 226         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 227     };
 228
 229     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 230 }
 231
 232 sub gen_pve_www_key {
 233
 234     return if -f $pvewww_key_fn;
 235
 236     eval {
 237         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 238     };
 239
 240     die "unable to generate pve www key:\n$@" if $@;
 241 }
 242
 243 sub update_serial {
 244     my ($serial) = @_;
 245
 246     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 247 }
 248
 249 sub gen_pve_ssl_cert {
 250     my ($force, $nodename, $ip) = @_;
 251
 252     die "no node name specified" if !$nodename;
 253     die "no IP specified" if !$ip;
 254
 255     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 256
 257     return if !$force && -f $pvessl_cert_fn;
 258
 259     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 260
 261     my $rc = PVE::INotify::read_file('resolvconf');
 262
 263     $names .= ",IP:$ip";
 264
 265     my $fqdn = $nodename;
 266
 267     $names .= ",DNS:$nodename";
 268
 269     if ($rc && $rc->{search}) {
 270         $fqdn = $nodename . "." . $rc->{search};
 271         $names .= ",DNS:$fqdn";
 272     }
 273
 274     my $sslconf = <<__EOD;
 275 RANDFILE = /root/.rnd
 276 extensions = v3_req
 277
 278 [ req ]
 279 default_bits = 2048
 280 distinguished_name = req_distinguished_name
 281 req_extensions = v3_req
 282 prompt = no
 283 string_mask = nombstr
 284
 285 [ req_distinguished_name ]
 286 organizationalUnitName = PVE Cluster Node
 287 organizationName = Proxmox Virtual Environment
 288 commonName = $fqdn
 289
 290 [ v3_req ]
 291 basicConstraints = CA:FALSE
 292 extendedKeyUsage = serverAuth
 293 subjectAltName = $names
 294 __EOD
 295
 296     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 297     my $fh = IO::File->new ($cfgfn, "w");
 298     print $fh $sslconf;
 299     close ($fh);
 300
 301     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 302     unlink $reqfn;
 303
 304     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 305     eval {
 306         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 307                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 308     };
 309
 310     if (my $err = $@) {
 311         unlink $reqfn;
 312         unlink $cfgfn;
 313         die "unable to generate pve certificate request:\n$err";
 314     }
 315
 316     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 317
 318     eval {
 319         # wrap openssl with faketime to prevent bug #904
 320         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
 321                         '-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
 322                         '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
 323                         '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
 324     };
 325
 326     if (my $err = $@) {
 327         unlink $reqfn;
 328         unlink $cfgfn;
 329         die "unable to generate pve ssl certificate:\n$err";
 330     }
 331
 332     unlink $cfgfn;
 333     unlink $reqfn;
 334 }
 335
 336 sub gen_pve_node_files {
 337     my ($nodename, $ip, $opt_force) = @_;
 338
 339     gen_local_dirs($nodename);
 340
 341     gen_auth_key();
 342
 343     # make sure we have a (cluster wide) secret
 344     # for CSRFR prevention
 345     gen_pve_www_key();
 346
 347     # make sure we have a (per node) private key
 348     gen_pve_ssl_key($nodename);
 349
 350     # make sure we have a CA
 351     my $force = gen_pveca_cert();
 352
 353     $force = 1 if $opt_force;
 354
 355     gen_pve_ssl_cert($force, $nodename, $ip);
 356 }
 357
 358 my $vzdump_cron_dummy = <<__EOD;
 359 # cluster wide vzdump cron schedule
 360 # Atomatically generated file - do not edit
 361
 362 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 363
 364 __EOD
 365
 366 sub gen_pve_vzdump_symlink {
 367
 368     my $filename = "/etc/pve/vzdump.cron";
 369
 370     my $link_fn = "/etc/cron.d/vzdump";
 371
 372     if ((-f $filename) && (! -l $link_fn)) {
 373         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 374         symlink($filename, $link_fn);
 375     }
 376 }
 377
 378 sub gen_pve_vzdump_files {
 379
 380     my $filename = "/etc/pve/vzdump.cron";
 381
 382     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 383         if ! -f $filename;
 384
 385     gen_pve_vzdump_symlink();
 386 };
 387
 388 my $versions = {};
 389 my $vmlist = {};
 390 my $clinfo = {};
 391
 392 my $ipcc_send_rec = sub {
 393     my ($msgid, $data) = @_;
 394
 395     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 396
 397     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 398
 399     return $res;
 400 };
 401
 402 my $ipcc_send_rec_json = sub {
 403     my ($msgid, $data) = @_;
 404
 405     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 406
 407     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 408
 409     return decode_json($res);
 410 };
 411
 412 my $ipcc_get_config = sub {
 413     my ($path) = @_;
 414
 415     my $bindata = pack "Z*", $path;
 416     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_CONFIG, $bindata);
 417     if (!defined($res)) {
 418         if ($! != 0) {
 419             return undef if $! == ENOENT;
 420             die "$!\n";
 421         }
 422         return '';
 423     }
 424
 425     return $res;
 426 };
 427
 428 my $ipcc_get_status = sub {
 429     my ($name, $nodename) = @_;
 430
 431     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 432     return PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_STATUS, $bindata);
 433 };
 434
 435 my $ipcc_update_status = sub {
 436     my ($name, $data) = @_;
 437
 438     my $raw = ref($data) ? encode_json($data) : $data;
 439     # update status
 440     my $bindata = pack "Z[256]Z*", $name, $raw;
 441
 442     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 443 };
 444
 445 my $ipcc_log = sub {
 446     my ($priority, $ident, $tag, $msg) = @_;
 447
 448     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 449     bytes::length($tag) + 1, $ident, $tag, $msg;
 450
 451     return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG, $bindata);
 452 };
 453
 454 my $ipcc_get_cluster_log = sub {
 455     my ($user, $max) = @_;
 456
 457     $max = 0 if !defined($max);
 458
 459     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 460     return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG, $bindata);
 461 };
 462
 463 my $ccache = {};
 464
 465 sub cfs_update {
 466     my ($fail) = @_;
 467     eval {
 468         my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION);
 469         #warn "GOT1: " . Dumper($res);
 470         die "no starttime\n" if !$res->{starttime};
 471
 472         if (!$res->{starttime} || !$versions->{starttime} ||
 473             $res->{starttime} != $versions->{starttime}) {
 474             #print "detected changed starttime\n";
 475             $vmlist = {};
 476             $clinfo = {};
 477             $ccache = {};
 478         }
 479
 480         $versions = $res;
 481     };
 482     my $err = $@;
 483     if ($err) {
 484         $versions = {};
 485         $vmlist = {};
 486         $clinfo = {};
 487         $ccache = {};
 488         die $err if $fail;
 489         warn $err;
 490     }
 491
 492     eval {
 493         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 494             #warn "detected new clinfo\n";
 495             $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO);
 496         }
 497     };
 498     $err = $@;
 499     if ($err) {
 500         $clinfo = {};
 501         die $err if $fail;
 502         warn $err;
 503     }
 504
 505     eval {
 506         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 507             #warn "detected new vmlist1\n";
 508             $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST);
 509         }
 510     };
 511     $err = $@;
 512     if ($err) {
 513         $vmlist = {};
 514         die $err if $fail;
 515         warn $err;
 516     }
 517 }
 518
 519 sub get_vmlist {
 520     return $vmlist;
 521 }
 522
 523 sub get_clinfo {
 524     return $clinfo;
 525 }
 526
 527 sub get_members {
 528     return $clinfo->{nodelist};
 529 }
 530
 531 sub get_nodelist {
 532     my $nodelist = $clinfo->{nodelist};
 533
 534     my $nodename = PVE::INotify::nodename();
 535
 536     if (!$nodelist || !$nodelist->{$nodename}) {
 537         return [ $nodename ];
 538     }
 539
 540     return [ keys %$nodelist ];
 541 }
 542
 543 # $data must be a chronological descending ordered array of tasks
 544 sub broadcast_tasklist {
 545     my ($data) = @_;
 546
 547     # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
 548     # from pmxcfs) - drop older items until we satisfy this constraint
 549     my $size = length(encode_json($data));
 550     while ($size >= (32 * 1024)) {
 551         pop @$data;
 552         $size = length(encode_json($data));
 553     }
 554
 555     eval {
 556         &$ipcc_update_status("tasklist", $data);
 557     };
 558
 559     warn $@ if $@;
 560 }
 561
 562 my $tasklistcache = {};
 563
 564 sub get_tasklist {
 565     my ($nodename) = @_;
 566
 567     my $kvstore = $versions->{kvstore} || {};
 568
 569     my $nodelist = get_nodelist();
 570
 571     my $res = [];
 572     foreach my $node (@$nodelist) {
 573         next if $nodename && ($nodename ne $node);
 574         eval {
 575             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 576             my $cd = $tasklistcache->{$node};
 577             if (!$cd || !$ver || !$cd->{version} ||
 578                 ($cd->{version} != $ver)) {
 579                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 580                 my $data = decode_json($raw);
 581                 push @$res, @$data;
 582                 $cd = $tasklistcache->{$node} = {
 583                     data => $data,
 584                     version => $ver,
 585                 };
 586             } elsif ($cd && $cd->{data}) {
 587                 push @$res, @{$cd->{data}};
 588             }
 589         };
 590         my $err = $@;
 591         syslog('err', $err) if $err;
 592     }
 593
 594     return $res;
 595 }
 596
 597 sub broadcast_rrd {
 598     my ($rrdid, $data) = @_;
 599
 600     eval {
 601         &$ipcc_update_status("rrd/$rrdid", $data);
 602     };
 603     my $err = $@;
 604
 605     warn $err if $err;
 606 }
 607
 608 my $last_rrd_dump = 0;
 609 my $last_rrd_data = "";
 610
 611 sub rrd_dump {
 612
 613     my $ctime = time();
 614
 615     my $diff = $ctime - $last_rrd_dump;
 616     if ($diff < 2) {
 617         return $last_rrd_data;
 618     }
 619
 620     my $raw;
 621     eval {
 622         $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP);
 623     };
 624     my $err = $@;
 625
 626     if ($err) {
 627         warn $err;
 628         return {};
 629     }
 630
 631     my $res = {};
 632
 633     if ($raw) {
 634         while ($raw =~ s/^(.*)\n//) {
 635             my ($key, @ela) = split(/:/, $1);
 636             next if !$key;
 637             next if !(scalar(@ela) > 1);
 638             $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
 639         }
 640     }
 641
 642     $last_rrd_dump = $ctime;
 643     $last_rrd_data = $res;
 644
 645     return $res;
 646 }
 647
 648 sub create_rrd_data {
 649     my ($rrdname, $timeframe, $cf) = @_;
 650
 651     my $rrddir = "/var/lib/rrdcached/db";
 652
 653     my $rrd = "$rrddir/$rrdname";
 654
 655     my $setup = {
 656         hour =>  [ 60, 70 ],
 657         day  =>  [ 60*30, 70 ],
 658         week =>  [ 60*180, 70 ],
 659         month => [ 60*720, 70 ],
 660         year =>  [ 60*10080, 70 ],
 661     };
 662
 663     my ($reso, $count) = @{$setup->{$timeframe}};
 664     my $ctime  = $reso*int(time()/$reso);
 665     my $req_start = $ctime - $reso*$count;
 666
 667     $cf = "AVERAGE" if !$cf;
 668
 669     my @args = (
 670         "-s" => $req_start,
 671         "-e" => $ctime - 1,
 672         "-r" => $reso,
 673         );
 674
 675     my $socket = "/var/run/rrdcached.sock";
 676     push @args, "--daemon" => "unix:$socket" if -S $socket;
 677
 678     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 679
 680     my $err = RRDs::error;
 681     die "RRD error: $err\n" if $err;
 682
 683     die "got wrong time resolution ($step != $reso)\n"
 684         if $step != $reso;
 685
 686     my $res = [];
 687     my $fields = scalar(@$names);
 688     for my $line (@$data) {
 689         my $entry = { 'time' => $start };
 690         $start += $step;
 691         for (my $i = 0; $i < $fields; $i++) {
 692             my $name = $names->[$i];
 693             if (defined(my $val = $line->[$i])) {
 694                 $entry->{$name} = $val;
 695             } else {
 696                 # leave empty fields undefined
 697                 # maybe make this configurable?
 698             }
 699         }
 700         push @$res, $entry;
 701     }
 702
 703     return $res;
 704 }
 705
 706 sub create_rrd_graph {
 707     my ($rrdname, $timeframe, $ds, $cf) = @_;
 708
 709     # Using RRD graph is clumsy - maybe it
 710     # is better to simply fetch the data, and do all display
 711     # related things with javascript (new extjs html5 graph library).
 712
 713     my $rrddir = "/var/lib/rrdcached/db";
 714
 715     my $rrd = "$rrddir/$rrdname";
 716
 717     my @ids = PVE::Tools::split_list($ds);
 718
 719     my $ds_txt = join('_', @ids);
 720
 721     my $filename = "${rrd}_${ds_txt}.png";
 722
 723     my $setup = {
 724         hour =>  [ 60, 60 ],
 725         day  =>  [ 60*30, 70 ],
 726         week =>  [ 60*180, 70 ],
 727         month => [ 60*720, 70 ],
 728         year =>  [ 60*10080, 70 ],
 729     };
 730
 731     my ($reso, $count) = @{$setup->{$timeframe}};
 732
 733     my @args = (
 734         "--imgformat" => "PNG",
 735         "--border" => 0,
 736         "--height" => 200,
 737         "--width" => 800,
 738         "--start" => - $reso*$count,
 739         "--end" => 'now' ,
 740         "--lower-limit" => 0,
 741         );
 742
 743     my $socket = "/var/run/rrdcached.sock";
 744     push @args, "--daemon" => "unix:$socket" if -S $socket;
 745
 746     my @coldef = ('#00ddff', '#ff0000');
 747
 748     $cf = "AVERAGE" if !$cf;
 749
 750     my $i = 0;
 751     foreach my $id (@ids) {
 752         my $col = $coldef[$i++] || die "fixme: no color definition";
 753         push @args, "DEF:${id}=$rrd:${id}:$cf";
 754         my $dataid = $id;
 755         if ($id eq 'cpu' || $id eq 'iowait') {
 756             push @args, "CDEF:${id}_per=${id},100,*";
 757             $dataid = "${id}_per";
 758         }
 759         push @args, "LINE2:${dataid}${col}:${id}";
 760     }
 761
 762     push @args, '--full-size-mode';
 763
 764     # we do not really store data into the file
 765     my $res = RRDs::graphv('-', @args);
 766
 767     my $err = RRDs::error;
 768     die "RRD error: $err\n" if $err;
 769
 770     return { filename => $filename, image => $res->{image} };
 771 }
 772
 773 # a fast way to read files (avoid fuse overhead)
 774 sub get_config {
 775     my ($path) = @_;
 776
 777     return &$ipcc_get_config($path);
 778 }
 779
 780 sub get_cluster_log {
 781     my ($user, $max) = @_;
 782
 783     return &$ipcc_get_cluster_log($user, $max);
 784 }
 785
 786 my $file_info = {};
 787
 788 sub cfs_register_file {
 789     my ($filename, $parser, $writer) = @_;
 790
 791     $observed->{$filename} || die "unknown file '$filename'";
 792
 793     die "file '$filename' already registered" if $file_info->{$filename};
 794
 795     $file_info->{$filename} = {
 796         parser => $parser,
 797         writer => $writer,
 798     };
 799 }
 800
 801 my $ccache_read = sub {
 802     my ($filename, $parser, $version) = @_;
 803
 804     $ccache->{$filename} = {} if !$ccache->{$filename};
 805
 806     my $ci = $ccache->{$filename};
 807
 808     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 809         # we always call the parser, even when the file does not exists
 810         # (in that case $data is undef)
 811         my $data = get_config($filename);
 812         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 813         $ci->{version} = $version;
 814     }
 815
 816     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 817
 818     return $res;
 819 };
 820
 821 sub cfs_file_version {
 822     my ($filename) = @_;
 823
 824     my $version;
 825     my $infotag;
 826     if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
 827         my ($type, $vmid) = ($1, $2);
 828         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 829             $version = $vmlist->{ids}->{$vmid}->{version};
 830         }
 831         $infotag = "/$type/";
 832     } else {
 833         $infotag = $filename;
 834         $version = $versions->{$filename};
 835     }
 836
 837     my $info = $file_info->{$infotag} ||
 838         die "unknown file type '$filename'\n";
 839
 840     return wantarray ? ($version, $info) : $version;
 841 }
 842
 843 sub cfs_read_file {
 844     my ($filename) = @_;
 845
 846     my ($version, $info) = cfs_file_version($filename);
 847     my $parser = $info->{parser};
 848
 849     return &$ccache_read($filename, $parser, $version);
 850 }
 851
 852 sub cfs_write_file {
 853     my ($filename, $data) = @_;
 854
 855     my ($version, $info) = cfs_file_version($filename);
 856
 857     my $writer = $info->{writer} || die "no writer defined";
 858
 859     my $fsname = "/etc/pve/$filename";
 860
 861     my $raw = &$writer($fsname, $data);
 862
 863     if (my $ci = $ccache->{$filename}) {
 864         $ci->{version} = undef;
 865     }
 866
 867     PVE::Tools::file_set_contents($fsname, $raw);
 868 }
 869
 870 my $cfs_lock = sub {
 871     my ($lockid, $timeout, $code, @param) = @_;
 872
 873     my $prev_alarm = alarm(0); # suspend outer alarm early
 874
 875     my $res;
 876     my $got_lock = 0;
 877
 878     # this timeout is for acquire the lock
 879     $timeout = 10 if !$timeout;
 880
 881     my $filename = "$lockdir/$lockid";
 882
 883     eval {
 884
 885         mkdir $lockdir;
 886
 887         if (! -d $lockdir) {
 888             die "pve cluster filesystem not online.\n";
 889         }
 890
 891         my $timeout_err = sub { die "got lock request timeout\n"; };
 892         local $SIG{ALRM} = $timeout_err;
 893
 894         while (1) {
 895             alarm ($timeout);
 896             $got_lock = mkdir($filename);
 897             $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
 898
 899             last if $got_lock;
 900
 901             $timeout_err->() if $timeout <= 0;
 902
 903             print STDERR "trying to acquire cfs lock '$lockid' ...\n";
 904             utime (0, 0, $filename); # cfs unlock request
 905             sleep(1);
 906         }
 907
 908         # fixed command timeout: cfs locks have a timeout of 120
 909         # using 60 gives us another 60 seconds to abort the task
 910         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 911         alarm(60);
 912
 913         cfs_update(); # make sure we read latest versions inside code()
 914
 915         $res = &$code(@param);
 916
 917         alarm(0);
 918     };
 919
 920     my $err = $@;
 921
 922     $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
 923
 924     rmdir $filename if $got_lock; # if we held the lock always unlock again
 925
 926     alarm($prev_alarm);
 927
 928     if ($err) {
 929         $@ = "error with cfs lock '$lockid': $err";
 930         return undef;
 931     }
 932
 933     $@ = undef;
 934
 935     return $res;
 936 };
 937
 938 sub cfs_lock_file {
 939     my ($filename, $timeout, $code, @param) = @_;
 940
 941     my $info = $observed->{$filename} || die "unknown file '$filename'";
 942
 943     my $lockid = "file-$filename";
 944     $lockid =~ s/[.\/]/_/g;
 945
 946     &$cfs_lock($lockid, $timeout, $code, @param);
 947 }
 948
 949 sub cfs_lock_storage {
 950     my ($storeid, $timeout, $code, @param) = @_;
 951
 952     my $lockid = "storage-$storeid";
 953
 954     &$cfs_lock($lockid, $timeout, $code, @param);
 955 }
 956
 957 sub cfs_lock_domain {
 958     my ($domainname, $timeout, $code, @param) = @_;
 959
 960     my $lockid = "domain-$domainname";
 961
 962     &$cfs_lock($lockid, $timeout, $code, @param);
 963 }
 964
 965 sub cfs_lock_acme {
 966     my ($account, $timeout, $code, @param) = @_;
 967
 968     my $lockid = "acme-$account";
 969
 970     &$cfs_lock($lockid, $timeout, $code, @param);
 971 }
 972
 973 sub cfs_lock_authkey {
 974     my ($timeout, $code, @param) = @_;
 975
 976     $cfs_lock->('authkey', $timeout, $code, @param);
 977 }
 978
 979 my $log_levels = {
 980     "emerg" => 0,
 981     "alert" => 1,
 982     "crit" => 2,
 983     "critical" => 2,
 984     "err" => 3,
 985     "error" => 3,
 986     "warn" => 4,
 987     "warning" => 4,
 988     "notice" => 5,
 989     "info" => 6,
 990     "debug" => 7,
 991 };
 992
 993 sub log_msg {
 994    my ($priority, $ident, $msg) = @_;
 995
 996    if (my $tmp = $log_levels->{$priority}) {
 997        $priority = $tmp;
 998    }
 999
1000    die "need numeric log priority" if $priority !~ /^\d+$/;
1001
1002    my $tag = PVE::SafeSyslog::tag();
1003
1004    $msg = "empty message" if !$msg;
1005
1006    $ident = "" if !$ident;
1007    $ident = encode("ascii", $ident,
1008                    sub { sprintf "\\u%04x", shift });
1009
1010    my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
1011
1012    if ($ident) {
1013        syslog($priority, "<%s> %s", $ident, $ascii);
1014    } else {
1015        syslog($priority, "%s", $ascii);
1016    }
1017
1018    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
1019
1020    syslog("err", "writing cluster log failed: $@") if $@;
1021 }
1022
1023 sub check_vmid_unused {
1024     my ($vmid, $noerr) = @_;
1025
1026     my $vmlist = get_vmlist();
1027
1028     my $d = $vmlist->{ids}->{$vmid};
1029     return 1 if !defined($d);
1030
1031     return undef if $noerr;
1032
1033     my $vmtypestr =  $d->{type} eq 'qemu' ? 'VM' : 'CT';
1034     die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
1035 }
1036
1037 sub check_node_exists {
1038     my ($nodename, $noerr) = @_;
1039
1040     my $nodelist = $clinfo->{nodelist};
1041     return 1 if $nodelist && $nodelist->{$nodename};
1042
1043     return undef if $noerr;
1044
1045     die "no such cluster node '$nodename'\n";
1046 }
1047
1048 # this is also used to get the IP of the local node
1049 sub remote_node_ip {
1050     my ($nodename, $noerr) = @_;
1051
1052     my $nodelist = $clinfo->{nodelist};
1053     if ($nodelist && $nodelist->{$nodename}) {
1054         if (my $ip = $nodelist->{$nodename}->{ip}) {
1055             return $ip if !wantarray;
1056             my $family = $nodelist->{$nodename}->{address_family};
1057             if (!$family) {
1058                 $nodelist->{$nodename}->{address_family} =
1059                     $family =
1060                     PVE::Tools::get_host_address_family($ip);
1061             }
1062             return wantarray ? ($ip, $family) : $ip;
1063         }
1064     }
1065
1066     # fallback: try to get IP by other means
1067     return PVE::Network::get_ip_from_hostname($nodename, $noerr);
1068 }
1069
1070 sub get_local_migration_ip {
1071     my ($migration_network, $noerr) = @_;
1072
1073     my $cidr = $migration_network;
1074
1075     if (!defined($cidr)) {
1076         my $dc_conf = cfs_read_file('datacenter.cfg');
1077         $cidr = $dc_conf->{migration}->{network}
1078           if defined($dc_conf->{migration}->{network});
1079     }
1080
1081     if (defined($cidr)) {
1082         my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
1083
1084         die "could not get migration ip: no IP address configured on local " .
1085             "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
1086
1087         die "could not get migration ip: multiple IP address configured for " .
1088             "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
1089
1090         return @$ips[0];
1091     }
1092
1093     return undef;
1094 };
1095
1096 # ssh related utility functions
1097
1098 sub ssh_merge_keys {
1099     # remove duplicate keys in $sshauthkeys
1100     # ssh-copy-id simply add keys, so the file can grow to large
1101
1102     my $data = '';
1103     if (-f $sshauthkeys) {
1104         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
1105         chomp($data);
1106     }
1107
1108     my $found_backup;
1109     if (-f $rootsshauthkeysbackup) {
1110         $data .= "\n";
1111         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
1112         chomp($data);
1113         $found_backup = 1;
1114     }
1115
1116     # always add ourself
1117     if (-f $ssh_rsa_id) {
1118         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
1119         chomp($pub);
1120         $data .= "\n$pub\n";
1121     }
1122
1123     my $newdata = "";
1124     my $vhash = {};
1125     my @lines = split(/\n/, $data);
1126     foreach my $line (@lines) {
1127         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1128             next if $vhash->{$3}++;
1129         }
1130         $newdata .= "$line\n";
1131     }
1132
1133     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
1134
1135     if ($found_backup && -l $rootsshauthkeys) {
1136         # everything went well, so we can remove the backup
1137         unlink $rootsshauthkeysbackup;
1138     }
1139 }
1140
1141 sub setup_sshd_config {
1142     my () = @_;
1143
1144     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
1145
1146     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1147
1148     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1149         chomp $conf;
1150         $conf .= "\nPermitRootLogin yes\n";
1151     }
1152
1153     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
1154
1155     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
1156 }
1157
1158 sub setup_rootsshconfig {
1159
1160     # create ssh key if it does not exist
1161     if (! -f $ssh_rsa_id) {
1162         mkdir '/root/.ssh/';
1163         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1164     }
1165
1166     # create ssh config if it does not exist
1167     if (! -f $rootsshconfig) {
1168         mkdir '/root/.ssh';
1169         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
1170             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
1171             # changed order to put AES before Chacha20 (most hardware has AESNI)
1172             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
1173             close($fh);
1174         }
1175     }
1176 }
1177
1178 sub setup_ssh_keys {
1179
1180     mkdir $authdir;
1181
1182     my $import_ok;
1183
1184     if (! -f $sshauthkeys) {
1185         my $old;
1186         if (-f $rootsshauthkeys) {
1187             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
1188         }
1189         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1190             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
1191             close($fh);
1192             $import_ok = 1;
1193         }
1194     }
1195
1196     warn "can't create shared ssh key database '$sshauthkeys'\n"
1197         if ! -f $sshauthkeys;
1198
1199     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
1200         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1201             warn "rename $rootsshauthkeys failed - $!\n";
1202         }
1203     }
1204
1205     if (! -l $rootsshauthkeys) {
1206         symlink $sshauthkeys, $rootsshauthkeys;
1207     }
1208
1209     if (! -l $rootsshauthkeys) {
1210         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1211     } else {
1212         unlink $rootsshauthkeysbackup if $import_ok;
1213     }
1214 }
1215
1216 sub ssh_unmerge_known_hosts {
1217     return if ! -l $sshglobalknownhosts;
1218
1219     my $old = '';
1220     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1221         if -f $sshknownhosts;
1222
1223     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1224 }
1225
1226 sub ssh_merge_known_hosts {
1227     my ($nodename, $ip_address, $createLink) = @_;
1228
1229     die "no node name specified" if !$nodename;
1230     die "no ip address specified" if !$ip_address;
1231
1232     # ssh lowercases hostnames (aliases) before comparision, so we need too
1233     $nodename = lc($nodename);
1234     $ip_address = lc($ip_address);
1235
1236     mkdir $authdir;
1237
1238     if (! -f $sshknownhosts) {
1239         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1240             close($fh);
1241         }
1242     }
1243
1244     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1245
1246     my $new = '';
1247
1248     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1249         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1250     }
1251
1252     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1253     # Note: file sometimes containe emty lines at start, so we use multiline match
1254     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
1255     $hostkey = $1;
1256
1257     my $data = '';
1258     my $vhash = {};
1259
1260     my $found_nodename;
1261     my $found_local_ip;
1262
1263     my $merge_line = sub {
1264         my ($line, $all) = @_;
1265
1266         return if $line =~ m/^\s*$/; # skip empty lines
1267         return if $line =~ m/^#/; # skip comments
1268
1269         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1270             my $key = $1;
1271             my $rsakey = $2;
1272             if (!$vhash->{$key}) {
1273                 $vhash->{$key} = 1;
1274                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1275                     my $salt = decode_base64($1);
1276                     my $digest = $2;
1277                     my $hmac = Digest::HMAC_SHA1->new($salt);
1278                     $hmac->add($nodename);
1279                     my $hd = $hmac->b64digest . '=';
1280                     if ($digest eq $hd) {
1281                         if ($rsakey eq $hostkey) {
1282                             $found_nodename = 1;
1283                             $data .= $line;
1284                         }
1285                         return;
1286                     }
1287                     $hmac = Digest::HMAC_SHA1->new($salt);
1288                     $hmac->add($ip_address);
1289                     $hd = $hmac->b64digest . '=';
1290                     if ($digest eq $hd) {
1291                         if ($rsakey eq $hostkey) {
1292                             $found_local_ip = 1;
1293                             $data .= $line;
1294                         }
1295                         return;
1296                     }
1297                 } else {
1298                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
1299                     if ($key eq $ip_address) {
1300                         $found_local_ip = 1 if $rsakey eq $hostkey;
1301                     } elsif ($key eq $nodename) {
1302                         $found_nodename = 1 if $rsakey eq $hostkey;
1303                     }
1304                 }
1305                 $data .= $line;
1306             }
1307         } elsif ($all) {
1308             $data .= $line;
1309         }
1310     };
1311
1312     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1313         my $line = "$2\n";
1314         &$merge_line($line, 1);
1315     }
1316
1317     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1318         my $line = "$2\n";
1319         &$merge_line($line);
1320     }
1321
1322     # add our own key if not already there
1323     $data .= "$nodename $hostkey\n" if !$found_nodename;
1324     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
1325
1326     PVE::Tools::file_set_contents($sshknownhosts, $data);
1327
1328     return if !$createLink;
1329
1330     unlink $sshglobalknownhosts;
1331     symlink $sshknownhosts, $sshglobalknownhosts;
1332
1333     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1334         if ! -l $sshglobalknownhosts;
1335
1336 }
1337
1338 my $migration_format = {
1339     type => {
1340         default_key => 1,
1341         type => 'string',
1342         enum => ['secure', 'insecure'],
1343         description => "Migration traffic is encrypted using an SSH tunnel by " .
1344           "default. On secure, completely private networks this can be " .
1345           "disabled to increase performance.",
1346         default => 'secure',
1347     },
1348     network => {
1349         optional => 1,
1350         type => 'string', format => 'CIDR',
1351         format_description => 'CIDR',
1352         description => "CIDR of the (sub) network that is used for migration."
1353     },
1354 };
1355
1356 my $ha_format = {
1357     shutdown_policy => {
1358         type => 'string',
1359         enum => ['freeze', 'failover', 'conditional'],
1360         description => "The policy for HA services on node shutdown. 'freeze' disables auto-recovery, 'failover' ensures recovery, 'conditional' recovers on poweroff and freezes on reboot. Running HA Services will always get stopped first on shutdown.",
1361         verbose_description => "Describes the policy for handling HA services on poweroff or reboot of a node. Freeze will always freeze services which are still located on the node on shutdown, those services won't be recovered by the HA manager. Failover will not mark the services as frozen and thus the services will get recovered to other nodes, if the shutdown node does not come up again quickly (< 1min). 'conditional' chooses automatically depending on the type of shutdown, i.e., on a reboot the service will be frozen but on a poweroff the service will stay as is, and thus get recovered after about 2 minutes.",
1362         default => 'conditional',
1363     }
1364 };
1365
1366 PVE::JSONSchema::register_format('mac-prefix', \&pve_verify_mac_prefix);
1367 sub pve_verify_mac_prefix {
1368     my ($mac_prefix, $noerr) = @_;
1369
1370     if ($mac_prefix !~ m/^[a-f0-9][02468ace](?::[a-f0-9]{2}){0,2}:?$/i) {
1371         return undef if $noerr;
1372         die "value is not a valid unicast MAC address prefix\n";
1373     }
1374     return $mac_prefix;
1375 }
1376
1377 our $u2f_format = {
1378     appid => {
1379         type => 'string',
1380         description => "U2F AppId URL override. Defaults to the origin.",
1381         format_description => 'APPID',
1382         optional => 1,
1383     },
1384     origin => {
1385         type => 'string',
1386         description => "U2F Origin override. Mostly useful for single nodes with a single URL.",
1387         format_description => 'URL',
1388         optional => 1,
1389     },
1390 };
1391
1392 my $datacenter_schema = {
1393     type => "object",
1394     additionalProperties => 0,
1395     properties => {
1396         keyboard => {
1397             optional => 1,
1398             type => 'string',
1399             description => "Default keybord layout for vnc server.",
1400             enum => PVE::Tools::kvmkeymaplist(),
1401         },
1402         language => {
1403             optional => 1,
1404             type => 'string',
1405             description => "Default GUI language.",
1406             enum => [
1407                 'zh_CN',
1408                 'zh_TW',
1409                 'ca',
1410                 'en',
1411                 'eu',
1412                 'fr',
1413                 'de',
1414                 'it',
1415                 'es',
1416                 'ja',
1417                 'nb',
1418                 'nn',
1419                 'fa',
1420                 'pl',
1421                 'pt_BR',
1422                 'ru',
1423                 'sl',
1424                 'sv',
1425                 'tr',
1426             ],
1427         },
1428         http_proxy => {
1429             optional => 1,
1430             type => 'string',
1431             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1432             pattern => "http://.*",
1433         },
1434         migration_unsecure => {
1435             optional => 1,
1436             type => 'boolean',
1437             description => "Migration is secure using SSH tunnel by default. " .
1438               "For secure private networks you can disable it to speed up " .
1439               "migration. Deprecated, use the 'migration' property instead!",
1440         },
1441         migration => {
1442             optional => 1,
1443             type => 'string', format => $migration_format,
1444             description => "For cluster wide migration settings.",
1445         },
1446         console => {
1447             optional => 1,
1448             type => 'string',
1449             description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), an HTML5 based vnc viewer (noVNC), or an HTML5 based console client (xtermjs). If the selected viewer is not available (e.g. SPICE not activated for the VM), the fallback is noVNC.",
1450             enum => ['applet', 'vv', 'html5', 'xtermjs'],
1451         },
1452         email_from => {
1453             optional => 1,
1454             type => 'string',
1455             format => 'email-opt',
1456             description => "Specify email address to send notification from (default is root@\$hostname)",
1457         },
1458         max_workers => {
1459             optional => 1,
1460             type => 'integer',
1461             minimum => 1,
1462             description => "Defines how many workers (per node) are maximal started ".
1463               " on actions like 'stopall VMs' or task from the ha-manager.",
1464         },
1465         fencing => {
1466             optional => 1,
1467             type => 'string',
1468             default => 'watchdog',
1469             enum => [ 'watchdog', 'hardware', 'both' ],
1470             description => "Set the fencing mode of the HA cluster. Hardware mode " .
1471               "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
1472               " With both all two modes are used." .
1473               "\n\nWARNING: 'hardware' and 'both' are EXPERIMENTAL & WIP",
1474         },
1475         ha => {
1476             optional => 1,
1477             type => 'string', format => $ha_format,
1478             description => "Cluster wide HA settings.",
1479         },
1480         mac_prefix => {
1481             optional => 1,
1482             type => 'string',
1483             format => 'mac-prefix',
1484             description => 'Prefix for autogenerated MAC addresses.',
1485         },
1486         bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
1487         u2f => {
1488             optional => 1,
1489             type => 'string',
1490             format => $u2f_format,
1491             description => 'u2f',
1492         },
1493     },
1494 };
1495
1496 # make schema accessible from outside (for documentation)
1497 sub get_datacenter_schema { return $datacenter_schema };
1498
1499 sub parse_datacenter_config {
1500     my ($filename, $raw) = @_;
1501
1502     my $res = PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
1503
1504     if (my $migration = $res->{migration}) {
1505         $res->{migration} = PVE::JSONSchema::parse_property_string($migration_format, $migration);
1506     }
1507
1508     if (my $ha = $res->{ha}) {
1509         $res->{ha} = PVE::JSONSchema::parse_property_string($ha_format, $ha);
1510     }
1511
1512     # for backwards compatibility only, new migration property has precedence
1513     if (defined($res->{migration_unsecure})) {
1514         if (defined($res->{migration}->{type})) {
1515             warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
1516               "set at same time! Ignore 'migration_unsecure'\n";
1517         } else {
1518             $res->{migration}->{type} = ($res->{migration_unsecure}) ? 'insecure' : 'secure';
1519         }
1520     }
1521
1522     # for backwards compatibility only, applet maps to html5
1523     if (defined($res->{console}) && $res->{console} eq 'applet') {
1524         $res->{console} = 'html5';
1525     }
1526
1527     return $res;
1528 }
1529
1530 sub write_datacenter_config {
1531     my ($filename, $cfg) = @_;
1532
1533     # map deprecated setting to new one
1534     if (defined($cfg->{migration_unsecure}) && !defined($cfg->{migration})) {
1535         my $migration_unsecure = delete $cfg->{migration_unsecure};
1536         $cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
1537     }
1538
1539     # map deprecated applet setting to html5
1540     if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
1541         $cfg->{console} = 'html5';
1542     }
1543
1544     if (ref($cfg->{migration})) {
1545         my $migration = $cfg->{migration};
1546         $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
1547     }
1548
1549     if (ref($cfg->{ha})) {
1550         my $ha = $cfg->{ha};
1551         $cfg->{ha} = PVE::JSONSchema::print_property_string($ha, $ha_format);
1552     }
1553
1554     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1555 }
1556
1557 cfs_register_file('datacenter.cfg',
1558                   \&parse_datacenter_config,
1559                   \&write_datacenter_config);
1560
1561 # X509 Certificate cache helper
1562
1563 my $cert_cache_nodes = {};
1564 my $cert_cache_timestamp = time();
1565 my $cert_cache_fingerprints = {};
1566
1567 sub update_cert_cache {
1568     my ($update_node, $clear) = @_;
1569
1570     syslog('info', "Clearing outdated entries from certificate cache")
1571         if $clear;
1572
1573     $cert_cache_timestamp = time() if !defined($update_node);
1574
1575     my $node_list = defined($update_node) ?
1576         [ $update_node ] : [ keys %$cert_cache_nodes ];
1577
1578     foreach my $node (@$node_list) {
1579         my $clear_old = sub {
1580             if (my $old_fp = $cert_cache_nodes->{$node}) {
1581                 # distrust old fingerprint
1582                 delete $cert_cache_fingerprints->{$old_fp};
1583                 # ensure reload on next proxied request
1584                 delete $cert_cache_nodes->{$node};
1585             }
1586         };
1587
1588         my $fp = eval { get_node_fingerprint($node) };
1589         if (my $err = $@) {
1590             warn "$err\n";
1591             &$clear_old() if $clear;
1592             next;
1593         }
1594
1595         my $old_fp = $cert_cache_nodes->{$node};
1596         $cert_cache_fingerprints->{$fp} = 1;
1597         $cert_cache_nodes->{$node} = $fp;
1598
1599         if (defined($old_fp) && $fp ne $old_fp) {
1600             delete $cert_cache_fingerprints->{$old_fp};
1601         }
1602     }
1603 }
1604
1605 # load and cache cert fingerprint once
1606 sub initialize_cert_cache {
1607     my ($node) = @_;
1608
1609     update_cert_cache($node)
1610         if defined($node) && !defined($cert_cache_nodes->{$node});
1611 }
1612
1613 sub read_ssl_cert_fingerprint {
1614     my ($cert_path) = @_;
1615
1616     my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
1617         or die "unable to read '$cert_path' - $!\n";
1618
1619     my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
1620     Net::SSLeay::BIO_free($bio);
1621
1622     die "unable to read certificate from '$cert_path'\n" if !$cert;
1623
1624     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1625     Net::SSLeay::X509_free($cert);
1626
1627     die "unable to get fingerprint for '$cert_path' - got empty value\n"
1628         if !defined($fp) || $fp eq '';
1629
1630     return $fp;
1631 }
1632
1633 sub get_node_fingerprint {
1634     my ($node) = @_;
1635
1636     my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
1637     my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
1638
1639     $cert_path = $custom_cert_path if -f $custom_cert_path;
1640
1641     return read_ssl_cert_fingerprint($cert_path);
1642 }
1643
1644
1645 sub check_cert_fingerprint {
1646     my ($cert) = @_;
1647
1648     # clear cache every 30 minutes at least
1649     update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
1650
1651     # get fingerprint of server certificate
1652     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1653     return 0 if !defined($fp) || $fp eq ''; # error
1654
1655     my $check = sub {
1656         for my $expected (keys %$cert_cache_fingerprints) {
1657             return 1 if $fp eq $expected;
1658         }
1659         return 0;
1660     };
1661
1662     return 1 if &$check();
1663
1664     # clear cache and retry at most once every minute
1665     if (time() - $cert_cache_timestamp >= 60) {
1666         syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
1667         update_cert_cache();
1668         return &$check();
1669     }
1670
1671     return 0;
1672 }
1673
1674 # bash completion helpers
1675
1676 sub complete_next_vmid {
1677
1678     my $vmlist = get_vmlist() || {};
1679     my $idlist = $vmlist->{ids} || {};
1680
1681     for (my $i = 100; $i < 10000; $i++) {
1682         return [$i] if !defined($idlist->{$i});
1683     }
1684
1685     return [];
1686 }
1687
1688 sub complete_vmid {
1689
1690     my $vmlist = get_vmlist();
1691     my $ids = $vmlist->{ids} || {};
1692
1693     return [ keys %$ids ];
1694 }
1695
1696 sub complete_local_vmid {
1697
1698     my $vmlist = get_vmlist();
1699     my $ids = $vmlist->{ids} || {};
1700
1701     my $nodename = PVE::INotify::nodename();
1702
1703     my $res = [];
1704     foreach my $vmid (keys %$ids) {
1705         my $d = $ids->{$vmid};
1706         next if !$d->{node} || $d->{node} ne $nodename;
1707         push @$res, $vmid;
1708     }
1709
1710     return $res;
1711 }
1712
1713 sub complete_migration_target {
1714
1715     my $res = [];
1716
1717     my $nodename = PVE::INotify::nodename();
1718
1719     my $nodelist = get_nodelist();
1720     foreach my $node (@$nodelist) {
1721         next if $node eq $nodename;
1722         push @$res, $node;
1723     }
1724
1725     return $res;
1726 }
1727
1728 sub get_ssh_info {
1729     my ($node, $network_cidr) = @_;
1730
1731     my $ip;
1732     if (defined($network_cidr)) {
1733         # Use mtunnel via to get the remote node's ip inside $network_cidr.
1734         # This goes over the regular network (iow. uses get_ssh_info() with
1735         # $network_cidr undefined.
1736         # FIXME: Use the REST API client for this after creating an API entry
1737         # for get_migration_ip.
1738         my $default_remote = get_ssh_info($node, undef);
1739         my $default_ssh = ssh_info_to_command($default_remote);
1740         my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
1741             '-migration_network', $network_cidr,
1742             '-get_migration_ip'
1743         ];
1744         PVE::Tools::run_command($cmd, outfunc => sub {
1745             my ($line) = @_;
1746             chomp $line;
1747             die "internal error: unexpected output from mtunnel\n"
1748                 if defined($ip);
1749             if ($line =~ /^ip: '(.*)'$/) {
1750                 $ip = $1;
1751             } else {
1752                 die "internal error: bad output from mtunnel\n"
1753                     if defined($ip);
1754             }
1755         });
1756         die "failed to get ip for node '$node' in network '$network_cidr'\n"
1757             if !defined($ip);
1758     } else {
1759         $ip = remote_node_ip($node);
1760     }
1761
1762     return {
1763         ip => $ip,
1764         name => $node,
1765         network => $network_cidr,
1766     };
1767 }
1768
1769 sub ssh_info_to_command_base {
1770     my ($info, @extra_options) = @_;
1771     return [
1772         '/usr/bin/ssh',
1773         '-e', 'none',
1774         '-o', 'BatchMode=yes',
1775         '-o', 'HostKeyAlias='.$info->{name},
1776         @extra_options
1777     ];
1778 }
1779
1780 sub ssh_info_to_command {
1781     my ($info, @extra_options) = @_;
1782     my $cmd = ssh_info_to_command_base($info, @extra_options);
1783     push @$cmd, "root\@$info->{ip}";
1784     return $cmd;
1785 }
1786
1787 sub assert_joinable {
1788     my ($ring0_addr, $ring1_addr, $force) = @_;
1789
1790     my $errors = '';
1791     my $error = sub { $errors .= "* $_[0]\n"; };
1792
1793     if (-f $authfile) {
1794         $error->("authentication key '$authfile' already exists");
1795     }
1796
1797     if (-f $clusterconf)  {
1798         $error->("cluster config '$clusterconf' already exists");
1799     }
1800
1801     my $vmlist = get_vmlist();
1802     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
1803         $error->("this host already contains virtual guests");
1804     }
1805
1806     if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
1807         $error->("corosync is already running, is this node already in a cluster?!");
1808     }
1809
1810     # check if corosync ring IPs are configured on the current nodes interfaces
1811     my $check_ip = sub {
1812         my $ip = shift // return;
1813         my $logid = shift;
1814         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
1815             my $host = $ip;
1816             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
1817             if ($@) {
1818                 $error->("$logid: cannot use '$host': $@\n") ;
1819                 return;
1820             }
1821         }
1822
1823         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
1824         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
1825
1826         $error->("$logid: cannot use IP '$ip', it must be configured exactly once on local node!\n")
1827             if (scalar(@$configured_ips) != 1);
1828     };
1829
1830     $check_ip->($ring0_addr, 'ring0');
1831     $check_ip->($ring1_addr, 'ring1');
1832
1833     if ($errors) {
1834         warn "detected the following error(s):\n$errors";
1835         die "Check if node may join a cluster failed!\n" if !$force;
1836     }
1837 }
1838
1839 # NOTE: filesystem must be offline here, no DB changes allowed
1840 my $backup_cfs_database = sub {
1841     my ($dbfile) = @_;
1842
1843     mkdir $dbbackupdir;
1844
1845     my $ctime = time();
1846     my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
1847
1848     print "backup old database to '$backup_fn'\n";
1849
1850     my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \ ">${backup_fn}"] ];
1851     run_command($cmd, 'errmsg' => "cannot backup old database\n");
1852
1853     my $maxfiles = 10; # purge older backup
1854     my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql.gz> ];
1855
1856     if ((my $count = scalar(@$backups)) > $maxfiles) {
1857         foreach my $f (@$backups[$maxfiles..$count-1]) {
1858             next if $f !~ m/^(\S+)$/; # untaint
1859             print "delete old backup '$1'\n";
1860             unlink $1;
1861         }
1862     }
1863 };
1864
1865 sub join {
1866     my ($param) = @_;
1867
1868     my $nodename = PVE::INotify::nodename();
1869
1870     setup_sshd_config();
1871     setup_rootsshconfig();
1872     setup_ssh_keys();
1873
1874     # check if we can join with the given parameters and current node state
1875     my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
1876     assert_joinable($ring0_addr, $ring1_addr, $param->{force});
1877
1878     # make sure known_hosts is on local filesystem
1879     ssh_unmerge_known_hosts();
1880
1881     my $host = $param->{hostname};
1882     my $local_ip_address = remote_node_ip($nodename);
1883
1884     my $conn_args = {
1885         username => 'root@pam',
1886         password => $param->{password},
1887         cookie_name => 'PVEAuthCookie',
1888         protocol => 'https',
1889         host => $host,
1890         port => 8006,
1891     };
1892
1893     if (my $fp = $param->{fingerprint}) {
1894         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
1895     } else {
1896         # API schema ensures that we can only get here from CLI handler
1897         $conn_args->{manual_verification} = 1;
1898     }
1899
1900     print "Establishing API connection with host '$host'\n";
1901
1902     my $conn = PVE::APIClient::LWP->new(%$conn_args);
1903     $conn->login();
1904
1905     # login raises an exception on failure, so if we get here we're good
1906     print "Login succeeded.\n";
1907
1908     my $args = {};
1909     $args->{force} = $param->{force} if defined($param->{force});
1910     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
1911     $args->{votes} = $param->{votes} if defined($param->{votes});
1912     $args->{ring0_addr} = $ring0_addr // $local_ip_address;
1913     $args->{ring1_addr} = $ring1_addr if defined($ring1_addr);
1914
1915     print "Request addition of this node\n";
1916     my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
1917
1918     print "Join request OK, finishing setup locally\n";
1919
1920     # added successfuly - now prepare local node
1921     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
1922 }
1923
1924 sub finish_join {
1925     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
1926
1927     mkdir "$localclusterdir";
1928     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
1929     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
1930
1931     print "stopping pve-cluster service\n";
1932     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
1933     run_command($cmd, errmsg => "can't stop pve-cluster service");
1934
1935     $backup_cfs_database->($dbfile);
1936     unlink $dbfile;
1937
1938     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
1939     run_command($cmd, errmsg => "starting pve-cluster failed");
1940
1941     # wait for quorum
1942     my $printqmsg = 1;
1943     while (!check_cfs_quorum(1)) {
1944         if ($printqmsg) {
1945             print "waiting for quorum...";
1946             STDOUT->flush();
1947             $printqmsg = 0;
1948         }
1949         sleep(1);
1950     }
1951     print "OK\n" if !$printqmsg;
1952
1953     updatecerts_and_ssh(1);
1954
1955     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
1956     run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
1957
1958     print "successfully added node '$nodename' to cluster.\n";
1959 }
1960
1961 sub updatecerts_and_ssh {
1962     my ($force_new_cert, $silent) = @_;
1963
1964     my $p = sub { print "$_[0]\n" if !$silent };
1965
1966     setup_rootsshconfig();
1967
1968     gen_pve_vzdump_symlink();
1969
1970     if (!check_cfs_quorum(1)) {
1971         return undef if $silent;
1972         die "no quorum - unable to update files\n";
1973     }
1974
1975     setup_ssh_keys();
1976
1977     my $nodename = PVE::INotify::nodename();
1978     my $local_ip_address = remote_node_ip($nodename);
1979
1980     $p->("(re)generate node files");
1981     $p->("generate new node certificate") if $force_new_cert;
1982     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
1983
1984     $p->("merge authorized SSH keys and known hosts");
1985     ssh_merge_keys();
1986     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
1987     gen_pve_vzdump_files();
1988 }
1989
1990 1;