data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EEXIST ENOENT);
   6 use File::stat qw();
   7 use Socket;
   8 use Storable qw(dclone);
   9 use IO::File;
  10 use MIME::Base64;
  11 use Digest::SHA;
  12 use Digest::HMAC_SHA1;
  13 use Net::SSLeay;
  14 use PVE::Tools qw(run_command);
  15 use PVE::INotify;
  16 use PVE::IPCC;
  17 use PVE::SafeSyslog;
  18 use PVE::JSONSchema;
  19 use PVE::Network;
  20 use PVE::Cluster::IPCConst;
  21 use JSON;
  22 use RRDs;
  23 use Encode;
  24 use UUID;
  25 use base 'Exporter';
  26
  27 our @EXPORT_OK = qw(
  28 cfs_read_file
  29 cfs_write_file
  30 cfs_register_file
  31 cfs_lock_file);
  32
  33 use Data::Dumper; # fixme: remove
  34
  35 # x509 certificate utils
  36
  37 my $basedir = "/etc/pve";
  38 my $authdir = "$basedir/priv";
  39 my $lockdir = "/etc/pve/priv/lock";
  40
  41 # cfs and corosync files
  42 my $dbfile = "/var/lib/pve-cluster/config.db";
  43 my $dbbackupdir = "/var/lib/pve-cluster/backup";
  44 my $localclusterdir = "/etc/corosync";
  45 my $localclusterconf = "$localclusterdir/corosync.conf";
  46 my $authfile = "$localclusterdir/authkey";
  47 my $clusterconf = "$basedir/corosync.conf";
  48
  49 my $authprivkeyfn = "$authdir/authkey.key";
  50 my $authpubkeyfn = "$basedir/authkey.pub";
  51 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  52 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  53 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  54 # this is just a secret accessable by the web browser
  55 # and is used for CSRF prevention
  56 my $pvewww_key_fn = "$basedir/pve-www.key";
  57
  58 # ssh related files
  59 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  60 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  61 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  62 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  63 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  64 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  65 my $sshd_config_fn = "/etc/ssh/sshd_config";
  66 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  67 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  68 my $rootsshconfig = "/root/.ssh/config";
  69
  70 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
  71 # observed files are the one we can get directly through IPCC, they are cached
  72 # using a computed version and only those can be used by the cfs_*_file methods
  73 my $observed = {
  74     'vzdump.cron' => 1,
  75     'storage.cfg' => 1,
  76     'datacenter.cfg' => 1,
  77     'replication.cfg' => 1,
  78     'corosync.conf' => 1,
  79     'corosync.conf.new' => 1,
  80     'user.cfg' => 1,
  81     'domains.cfg' => 1,
  82     'priv/shadow.cfg' => 1,
  83     '/qemu-server/' => 1,
  84     '/openvz/' => 1,
  85     '/lxc/' => 1,
  86     'ha/crm_commands' => 1,
  87     'ha/manager_status' => 1,
  88     'ha/resources.cfg' => 1,
  89     'ha/groups.cfg' => 1,
  90     'ha/fence.cfg' => 1,
  91     'status.cfg' => 1,
  92     'ceph.conf' => 1,
  93 };
  94
  95 # only write output if something fails
  96 sub run_silent_cmd {
  97     my ($cmd) = @_;
  98
  99     my $outbuf = '';
 100     my $record = sub { $outbuf .= shift . "\n"; };
 101
 102     eval { run_command($cmd, outfunc => $record, errfunc => $record) };
 103
 104     if (my $err = $@) {
 105         print STDERR $outbuf;
 106         die $err;
 107     }
 108 }
 109
 110 sub check_cfs_quorum {
 111     my ($noerr) = @_;
 112
 113     # note: -w filename always return 1 for root, so wee need
 114     # to use File::lstat here
 115     my $st = File::stat::lstat("$basedir/local");
 116     my $quorate = ($st && (($st->mode & 0200) != 0));
 117
 118     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 119
 120     return $quorate;
 121 }
 122
 123 sub check_cfs_is_mounted {
 124     my ($noerr) = @_;
 125
 126     my $res = -l "$basedir/local";
 127
 128     die "pve configuration filesystem not mounted\n"
 129         if !$res && !$noerr;
 130
 131     return $res;
 132 }
 133
 134 sub gen_local_dirs {
 135     my ($nodename) = @_;
 136
 137     check_cfs_is_mounted();
 138
 139     my @required_dirs = (
 140         "$basedir/priv",
 141         "$basedir/nodes",
 142         "$basedir/nodes/$nodename",
 143         "$basedir/nodes/$nodename/lxc",
 144         "$basedir/nodes/$nodename/qemu-server",
 145         "$basedir/nodes/$nodename/openvz",
 146         "$basedir/nodes/$nodename/priv");
 147
 148     foreach my $dir (@required_dirs) {
 149         if (! -d $dir) {
 150             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 151         }
 152     }
 153 }
 154
 155 sub gen_auth_key {
 156
 157     return if -f "$authprivkeyfn";
 158
 159     check_cfs_is_mounted();
 160
 161     cfs_lock_authkey(undef, sub {
 162         mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
 163
 164         run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 165
 166         run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 167     });
 168
 169     die "$@\n" if $@;
 170 }
 171
 172 sub gen_pveca_key {
 173
 174     return if -f $pveca_key_fn;
 175
 176     eval {
 177         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 178     };
 179
 180     die "unable to generate pve ca key:\n$@" if $@;
 181 }
 182
 183 sub gen_pveca_cert {
 184
 185     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 186         return 0;
 187     }
 188
 189     gen_pveca_key();
 190
 191     # we try to generate an unique 'subject' to avoid browser problems
 192     # (reused serial numbers, ..)
 193     my $uuid;
 194     UUID::generate($uuid);
 195     my $uuid_str;
 196     UUID::unparse($uuid, $uuid_str);
 197
 198     eval {
 199         # wrap openssl with faketime to prevent bug #904
 200         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 201                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 202                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 203                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 204     };
 205
 206     die "generating pve root certificate failed:\n$@" if $@;
 207
 208     return 1;
 209 }
 210
 211 sub gen_pve_ssl_key {
 212     my ($nodename) = @_;
 213
 214     die "no node name specified" if !$nodename;
 215
 216     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 217
 218     return if -f $pvessl_key_fn;
 219
 220     eval {
 221         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 222     };
 223
 224     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 225 }
 226
 227 sub gen_pve_www_key {
 228
 229     return if -f $pvewww_key_fn;
 230
 231     eval {
 232         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 233     };
 234
 235     die "unable to generate pve www key:\n$@" if $@;
 236 }
 237
 238 sub update_serial {
 239     my ($serial) = @_;
 240
 241     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 242 }
 243
 244 sub gen_pve_ssl_cert {
 245     my ($force, $nodename, $ip) = @_;
 246
 247     die "no node name specified" if !$nodename;
 248     die "no IP specified" if !$ip;
 249
 250     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 251
 252     return if !$force && -f $pvessl_cert_fn;
 253
 254     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 255
 256     my $rc = PVE::INotify::read_file('resolvconf');
 257
 258     $names .= ",IP:$ip";
 259
 260     my $fqdn = $nodename;
 261
 262     $names .= ",DNS:$nodename";
 263
 264     if ($rc && $rc->{search}) {
 265         $fqdn = $nodename . "." . $rc->{search};
 266         $names .= ",DNS:$fqdn";
 267     }
 268
 269     my $sslconf = <<__EOD;
 270 RANDFILE = /root/.rnd
 271 extensions = v3_req
 272
 273 [ req ]
 274 default_bits = 2048
 275 distinguished_name = req_distinguished_name
 276 req_extensions = v3_req
 277 prompt = no
 278 string_mask = nombstr
 279
 280 [ req_distinguished_name ]
 281 organizationalUnitName = PVE Cluster Node
 282 organizationName = Proxmox Virtual Environment
 283 commonName = $fqdn
 284
 285 [ v3_req ]
 286 basicConstraints = CA:FALSE
 287 extendedKeyUsage = serverAuth
 288 subjectAltName = $names
 289 __EOD
 290
 291     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 292     my $fh = IO::File->new ($cfgfn, "w");
 293     print $fh $sslconf;
 294     close ($fh);
 295
 296     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 297     unlink $reqfn;
 298
 299     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 300     eval {
 301         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 302                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 303     };
 304
 305     if (my $err = $@) {
 306         unlink $reqfn;
 307         unlink $cfgfn;
 308         die "unable to generate pve certificate request:\n$err";
 309     }
 310
 311     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 312
 313     eval {
 314         # wrap openssl with faketime to prevent bug #904
 315         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
 316                         '-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
 317                         '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
 318                         '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
 319     };
 320
 321     if (my $err = $@) {
 322         unlink $reqfn;
 323         unlink $cfgfn;
 324         die "unable to generate pve ssl certificate:\n$err";
 325     }
 326
 327     unlink $cfgfn;
 328     unlink $reqfn;
 329 }
 330
 331 sub gen_pve_node_files {
 332     my ($nodename, $ip, $opt_force) = @_;
 333
 334     gen_local_dirs($nodename);
 335
 336     gen_auth_key();
 337
 338     # make sure we have a (cluster wide) secret
 339     # for CSRFR prevention
 340     gen_pve_www_key();
 341
 342     # make sure we have a (per node) private key
 343     gen_pve_ssl_key($nodename);
 344
 345     # make sure we have a CA
 346     my $force = gen_pveca_cert();
 347
 348     $force = 1 if $opt_force;
 349
 350     gen_pve_ssl_cert($force, $nodename, $ip);
 351 }
 352
 353 my $vzdump_cron_dummy = <<__EOD;
 354 # cluster wide vzdump cron schedule
 355 # Atomatically generated file - do not edit
 356
 357 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 358
 359 __EOD
 360
 361 sub gen_pve_vzdump_symlink {
 362
 363     my $filename = "/etc/pve/vzdump.cron";
 364
 365     my $link_fn = "/etc/cron.d/vzdump";
 366
 367     if ((-f $filename) && (! -l $link_fn)) {
 368         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 369         symlink($filename, $link_fn);
 370     }
 371 }
 372
 373 sub gen_pve_vzdump_files {
 374
 375     my $filename = "/etc/pve/vzdump.cron";
 376
 377     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 378         if ! -f $filename;
 379
 380     gen_pve_vzdump_symlink();
 381 };
 382
 383 my $versions = {};
 384 my $vmlist = {};
 385 my $clinfo = {};
 386
 387 my $ipcc_send_rec = sub {
 388     my ($msgid, $data) = @_;
 389
 390     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 391
 392     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 393
 394     return $res;
 395 };
 396
 397 my $ipcc_send_rec_json = sub {
 398     my ($msgid, $data) = @_;
 399
 400     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 401
 402     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 403
 404     return decode_json($res);
 405 };
 406
 407 my $ipcc_get_config = sub {
 408     my ($path) = @_;
 409
 410     my $bindata = pack "Z*", $path;
 411     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_CONFIG, $bindata);
 412     if (!defined($res)) {
 413         if ($! != 0) {
 414             return undef if $! == ENOENT;
 415             die "$!\n";
 416         }
 417         return '';
 418     }
 419
 420     return $res;
 421 };
 422
 423 my $ipcc_get_status = sub {
 424     my ($name, $nodename) = @_;
 425
 426     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 427     return PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_STATUS, $bindata);
 428 };
 429
 430 my $ipcc_update_status = sub {
 431     my ($name, $data) = @_;
 432
 433     my $raw = ref($data) ? encode_json($data) : $data;
 434     # update status
 435     my $bindata = pack "Z[256]Z*", $name, $raw;
 436
 437     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 438 };
 439
 440 my $ipcc_log = sub {
 441     my ($priority, $ident, $tag, $msg) = @_;
 442
 443     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 444     bytes::length($tag) + 1, $ident, $tag, $msg;
 445
 446     return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG, $bindata);
 447 };
 448
 449 my $ipcc_get_cluster_log = sub {
 450     my ($user, $max) = @_;
 451
 452     $max = 0 if !defined($max);
 453
 454     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 455     return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG, $bindata);
 456 };
 457
 458 my $ccache = {};
 459
 460 sub cfs_update {
 461     my ($fail) = @_;
 462     eval {
 463         my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION);
 464         #warn "GOT1: " . Dumper($res);
 465         die "no starttime\n" if !$res->{starttime};
 466
 467         if (!$res->{starttime} || !$versions->{starttime} ||
 468             $res->{starttime} != $versions->{starttime}) {
 469             #print "detected changed starttime\n";
 470             $vmlist = {};
 471             $clinfo = {};
 472             $ccache = {};
 473         }
 474
 475         $versions = $res;
 476     };
 477     my $err = $@;
 478     if ($err) {
 479         $versions = {};
 480         $vmlist = {};
 481         $clinfo = {};
 482         $ccache = {};
 483         die $err if $fail;
 484         warn $err;
 485     }
 486
 487     eval {
 488         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 489             #warn "detected new clinfo\n";
 490             $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO);
 491         }
 492     };
 493     $err = $@;
 494     if ($err) {
 495         $clinfo = {};
 496         die $err if $fail;
 497         warn $err;
 498     }
 499
 500     eval {
 501         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 502             #warn "detected new vmlist1\n";
 503             $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST);
 504         }
 505     };
 506     $err = $@;
 507     if ($err) {
 508         $vmlist = {};
 509         die $err if $fail;
 510         warn $err;
 511     }
 512 }
 513
 514 sub get_vmlist {
 515     return $vmlist;
 516 }
 517
 518 sub get_clinfo {
 519     return $clinfo;
 520 }
 521
 522 sub get_members {
 523     return $clinfo->{nodelist};
 524 }
 525
 526 sub get_nodelist {
 527     my $nodelist = $clinfo->{nodelist};
 528
 529     my $nodename = PVE::INotify::nodename();
 530
 531     if (!$nodelist || !$nodelist->{$nodename}) {
 532         return [ $nodename ];
 533     }
 534
 535     return [ keys %$nodelist ];
 536 }
 537
 538 # $data must be a chronological descending ordered array of tasks
 539 sub broadcast_tasklist {
 540     my ($data) = @_;
 541
 542     # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
 543     # from pmxcfs) - drop older items until we satisfy this constraint
 544     my $size = length(encode_json($data));
 545     while ($size >= (32 * 1024)) {
 546         pop @$data;
 547         $size = length(encode_json($data));
 548     }
 549
 550     eval {
 551         &$ipcc_update_status("tasklist", $data);
 552     };
 553
 554     warn $@ if $@;
 555 }
 556
 557 my $tasklistcache = {};
 558
 559 sub get_tasklist {
 560     my ($nodename) = @_;
 561
 562     my $kvstore = $versions->{kvstore} || {};
 563
 564     my $nodelist = get_nodelist();
 565
 566     my $res = [];
 567     foreach my $node (@$nodelist) {
 568         next if $nodename && ($nodename ne $node);
 569         eval {
 570             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 571             my $cd = $tasklistcache->{$node};
 572             if (!$cd || !$ver || !$cd->{version} ||
 573                 ($cd->{version} != $ver)) {
 574                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 575                 my $data = decode_json($raw);
 576                 push @$res, @$data;
 577                 $cd = $tasklistcache->{$node} = {
 578                     data => $data,
 579                     version => $ver,
 580                 };
 581             } elsif ($cd && $cd->{data}) {
 582                 push @$res, @{$cd->{data}};
 583             }
 584         };
 585         my $err = $@;
 586         syslog('err', $err) if $err;
 587     }
 588
 589     return $res;
 590 }
 591
 592 sub broadcast_rrd {
 593     my ($rrdid, $data) = @_;
 594
 595     eval {
 596         &$ipcc_update_status("rrd/$rrdid", $data);
 597     };
 598     my $err = $@;
 599
 600     warn $err if $err;
 601 }
 602
 603 my $last_rrd_dump = 0;
 604 my $last_rrd_data = "";
 605
 606 sub rrd_dump {
 607
 608     my $ctime = time();
 609
 610     my $diff = $ctime - $last_rrd_dump;
 611     if ($diff < 2) {
 612         return $last_rrd_data;
 613     }
 614
 615     my $raw;
 616     eval {
 617         $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP);
 618     };
 619     my $err = $@;
 620
 621     if ($err) {
 622         warn $err;
 623         return {};
 624     }
 625
 626     my $res = {};
 627
 628     if ($raw) {
 629         while ($raw =~ s/^(.*)\n//) {
 630             my ($key, @ela) = split(/:/, $1);
 631             next if !$key;
 632             next if !(scalar(@ela) > 1);
 633             $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
 634         }
 635     }
 636
 637     $last_rrd_dump = $ctime;
 638     $last_rrd_data = $res;
 639
 640     return $res;
 641 }
 642
 643 sub create_rrd_data {
 644     my ($rrdname, $timeframe, $cf) = @_;
 645
 646     my $rrddir = "/var/lib/rrdcached/db";
 647
 648     my $rrd = "$rrddir/$rrdname";
 649
 650     my $setup = {
 651         hour =>  [ 60, 70 ],
 652         day  =>  [ 60*30, 70 ],
 653         week =>  [ 60*180, 70 ],
 654         month => [ 60*720, 70 ],
 655         year =>  [ 60*10080, 70 ],
 656     };
 657
 658     my ($reso, $count) = @{$setup->{$timeframe}};
 659     my $ctime  = $reso*int(time()/$reso);
 660     my $req_start = $ctime - $reso*$count;
 661
 662     $cf = "AVERAGE" if !$cf;
 663
 664     my @args = (
 665         "-s" => $req_start,
 666         "-e" => $ctime - 1,
 667         "-r" => $reso,
 668         );
 669
 670     my $socket = "/var/run/rrdcached.sock";
 671     push @args, "--daemon" => "unix:$socket" if -S $socket;
 672
 673     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 674
 675     my $err = RRDs::error;
 676     die "RRD error: $err\n" if $err;
 677
 678     die "got wrong time resolution ($step != $reso)\n"
 679         if $step != $reso;
 680
 681     my $res = [];
 682     my $fields = scalar(@$names);
 683     for my $line (@$data) {
 684         my $entry = { 'time' => $start };
 685         $start += $step;
 686         for (my $i = 0; $i < $fields; $i++) {
 687             my $name = $names->[$i];
 688             if (defined(my $val = $line->[$i])) {
 689                 $entry->{$name} = $val;
 690             } else {
 691                 # leave empty fields undefined
 692                 # maybe make this configurable?
 693             }
 694         }
 695         push @$res, $entry;
 696     }
 697
 698     return $res;
 699 }
 700
 701 sub create_rrd_graph {
 702     my ($rrdname, $timeframe, $ds, $cf) = @_;
 703
 704     # Using RRD graph is clumsy - maybe it
 705     # is better to simply fetch the data, and do all display
 706     # related things with javascript (new extjs html5 graph library).
 707
 708     my $rrddir = "/var/lib/rrdcached/db";
 709
 710     my $rrd = "$rrddir/$rrdname";
 711
 712     my @ids = PVE::Tools::split_list($ds);
 713
 714     my $ds_txt = join('_', @ids);
 715
 716     my $filename = "${rrd}_${ds_txt}.png";
 717
 718     my $setup = {
 719         hour =>  [ 60, 60 ],
 720         day  =>  [ 60*30, 70 ],
 721         week =>  [ 60*180, 70 ],
 722         month => [ 60*720, 70 ],
 723         year =>  [ 60*10080, 70 ],
 724     };
 725
 726     my ($reso, $count) = @{$setup->{$timeframe}};
 727
 728     my @args = (
 729         "--imgformat" => "PNG",
 730         "--border" => 0,
 731         "--height" => 200,
 732         "--width" => 800,
 733         "--start" => - $reso*$count,
 734         "--end" => 'now' ,
 735         "--lower-limit" => 0,
 736         );
 737
 738     my $socket = "/var/run/rrdcached.sock";
 739     push @args, "--daemon" => "unix:$socket" if -S $socket;
 740
 741     my @coldef = ('#00ddff', '#ff0000');
 742
 743     $cf = "AVERAGE" if !$cf;
 744
 745     my $i = 0;
 746     foreach my $id (@ids) {
 747         my $col = $coldef[$i++] || die "fixme: no color definition";
 748         push @args, "DEF:${id}=$rrd:${id}:$cf";
 749         my $dataid = $id;
 750         if ($id eq 'cpu' || $id eq 'iowait') {
 751             push @args, "CDEF:${id}_per=${id},100,*";
 752             $dataid = "${id}_per";
 753         }
 754         push @args, "LINE2:${dataid}${col}:${id}";
 755     }
 756
 757     push @args, '--full-size-mode';
 758
 759     # we do not really store data into the file
 760     my $res = RRDs::graphv('-', @args);
 761
 762     my $err = RRDs::error;
 763     die "RRD error: $err\n" if $err;
 764
 765     return { filename => $filename, image => $res->{image} };
 766 }
 767
 768 # a fast way to read files (avoid fuse overhead)
 769 sub get_config {
 770     my ($path) = @_;
 771
 772     return &$ipcc_get_config($path);
 773 }
 774
 775 sub get_cluster_log {
 776     my ($user, $max) = @_;
 777
 778     return &$ipcc_get_cluster_log($user, $max);
 779 }
 780
 781 my $file_info = {};
 782
 783 sub cfs_register_file {
 784     my ($filename, $parser, $writer) = @_;
 785
 786     $observed->{$filename} || die "unknown file '$filename'";
 787
 788     die "file '$filename' already registered" if $file_info->{$filename};
 789
 790     $file_info->{$filename} = {
 791         parser => $parser,
 792         writer => $writer,
 793     };
 794 }
 795
 796 my $ccache_read = sub {
 797     my ($filename, $parser, $version) = @_;
 798
 799     $ccache->{$filename} = {} if !$ccache->{$filename};
 800
 801     my $ci = $ccache->{$filename};
 802
 803     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 804         # we always call the parser, even when the file does not exists
 805         # (in that case $data is undef)
 806         my $data = get_config($filename);
 807         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 808         $ci->{version} = $version;
 809     }
 810
 811     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 812
 813     return $res;
 814 };
 815
 816 sub cfs_file_version {
 817     my ($filename) = @_;
 818
 819     my $version;
 820     my $infotag;
 821     if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
 822         my ($type, $vmid) = ($1, $2);
 823         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 824             $version = $vmlist->{ids}->{$vmid}->{version};
 825         }
 826         $infotag = "/$type/";
 827     } else {
 828         $infotag = $filename;
 829         $version = $versions->{$filename};
 830     }
 831
 832     my $info = $file_info->{$infotag} ||
 833         die "unknown file type '$filename'\n";
 834
 835     return wantarray ? ($version, $info) : $version;
 836 }
 837
 838 sub cfs_read_file {
 839     my ($filename) = @_;
 840
 841     my ($version, $info) = cfs_file_version($filename);
 842     my $parser = $info->{parser};
 843
 844     return &$ccache_read($filename, $parser, $version);
 845 }
 846
 847 sub cfs_write_file {
 848     my ($filename, $data) = @_;
 849
 850     my ($version, $info) = cfs_file_version($filename);
 851
 852     my $writer = $info->{writer} || die "no writer defined";
 853
 854     my $fsname = "/etc/pve/$filename";
 855
 856     my $raw = &$writer($fsname, $data);
 857
 858     if (my $ci = $ccache->{$filename}) {
 859         $ci->{version} = undef;
 860     }
 861
 862     PVE::Tools::file_set_contents($fsname, $raw);
 863 }
 864
 865 my $cfs_lock = sub {
 866     my ($lockid, $timeout, $code, @param) = @_;
 867
 868     my $prev_alarm = alarm(0); # suspend outer alarm early
 869
 870     my $res;
 871     my $got_lock = 0;
 872
 873     # this timeout is for acquire the lock
 874     $timeout = 10 if !$timeout;
 875
 876     my $filename = "$lockdir/$lockid";
 877
 878     eval {
 879
 880         mkdir $lockdir;
 881
 882         if (! -d $lockdir) {
 883             die "pve cluster filesystem not online.\n";
 884         }
 885
 886         my $timeout_err = sub { die "got lock request timeout\n"; };
 887         local $SIG{ALRM} = $timeout_err;
 888
 889         while (1) {
 890             alarm ($timeout);
 891             $got_lock = mkdir($filename);
 892             $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
 893
 894             last if $got_lock;
 895
 896             $timeout_err->() if $timeout <= 0;
 897
 898             print STDERR "trying to acquire cfs lock '$lockid' ...\n";
 899             utime (0, 0, $filename); # cfs unlock request
 900             sleep(1);
 901         }
 902
 903         # fixed command timeout: cfs locks have a timeout of 120
 904         # using 60 gives us another 60 seconds to abort the task
 905         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 906         alarm(60);
 907
 908         cfs_update(); # make sure we read latest versions inside code()
 909
 910         $res = &$code(@param);
 911
 912         alarm(0);
 913     };
 914
 915     my $err = $@;
 916
 917     $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
 918
 919     rmdir $filename if $got_lock; # if we held the lock always unlock again
 920
 921     alarm($prev_alarm);
 922
 923     if ($err) {
 924         $@ = "error with cfs lock '$lockid': $err";
 925         return undef;
 926     }
 927
 928     $@ = undef;
 929
 930     return $res;
 931 };
 932
 933 sub cfs_lock_file {
 934     my ($filename, $timeout, $code, @param) = @_;
 935
 936     my $info = $observed->{$filename} || die "unknown file '$filename'";
 937
 938     my $lockid = "file-$filename";
 939     $lockid =~ s/[.\/]/_/g;
 940
 941     &$cfs_lock($lockid, $timeout, $code, @param);
 942 }
 943
 944 sub cfs_lock_storage {
 945     my ($storeid, $timeout, $code, @param) = @_;
 946
 947     my $lockid = "storage-$storeid";
 948
 949     &$cfs_lock($lockid, $timeout, $code, @param);
 950 }
 951
 952 sub cfs_lock_domain {
 953     my ($domainname, $timeout, $code, @param) = @_;
 954
 955     my $lockid = "domain-$domainname";
 956
 957     &$cfs_lock($lockid, $timeout, $code, @param);
 958 }
 959
 960 sub cfs_lock_acme {
 961     my ($account, $timeout, $code, @param) = @_;
 962
 963     my $lockid = "acme-$account";
 964
 965     &$cfs_lock($lockid, $timeout, $code, @param);
 966 }
 967
 968 sub cfs_lock_authkey {
 969     my ($timeout, $code, @param) = @_;
 970
 971     $cfs_lock->('authkey', $timeout, $code, @param);
 972 }
 973
 974 my $log_levels = {
 975     "emerg" => 0,
 976     "alert" => 1,
 977     "crit" => 2,
 978     "critical" => 2,
 979     "err" => 3,
 980     "error" => 3,
 981     "warn" => 4,
 982     "warning" => 4,
 983     "notice" => 5,
 984     "info" => 6,
 985     "debug" => 7,
 986 };
 987
 988 sub log_msg {
 989    my ($priority, $ident, $msg) = @_;
 990
 991    if (my $tmp = $log_levels->{$priority}) {
 992        $priority = $tmp;
 993    }
 994
 995    die "need numeric log priority" if $priority !~ /^\d+$/;
 996
 997    my $tag = PVE::SafeSyslog::tag();
 998
 999    $msg = "empty message" if !$msg;
1000
1001    $ident = "" if !$ident;
1002    $ident = encode("ascii", $ident,
1003                    sub { sprintf "\\u%04x", shift });
1004
1005    my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
1006
1007    if ($ident) {
1008        syslog($priority, "<%s> %s", $ident, $ascii);
1009    } else {
1010        syslog($priority, "%s", $ascii);
1011    }
1012
1013    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
1014
1015    syslog("err", "writing cluster log failed: $@") if $@;
1016 }
1017
1018 sub check_vmid_unused {
1019     my ($vmid, $noerr) = @_;
1020
1021     my $vmlist = get_vmlist();
1022
1023     my $d = $vmlist->{ids}->{$vmid};
1024     return 1 if !defined($d);
1025
1026     return undef if $noerr;
1027
1028     my $vmtypestr =  $d->{type} eq 'qemu' ? 'VM' : 'CT';
1029     die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
1030 }
1031
1032 sub check_node_exists {
1033     my ($nodename, $noerr) = @_;
1034
1035     my $nodelist = $clinfo->{nodelist};
1036     return 1 if $nodelist && $nodelist->{$nodename};
1037
1038     return undef if $noerr;
1039
1040     die "no such cluster node '$nodename'\n";
1041 }
1042
1043 # this is also used to get the IP of the local node
1044 sub remote_node_ip {
1045     my ($nodename, $noerr) = @_;
1046
1047     my $nodelist = $clinfo->{nodelist};
1048     if ($nodelist && $nodelist->{$nodename}) {
1049         if (my $ip = $nodelist->{$nodename}->{ip}) {
1050             return $ip if !wantarray;
1051             my $family = $nodelist->{$nodename}->{address_family};
1052             if (!$family) {
1053                 $nodelist->{$nodename}->{address_family} =
1054                     $family =
1055                     PVE::Tools::get_host_address_family($ip);
1056             }
1057             return wantarray ? ($ip, $family) : $ip;
1058         }
1059     }
1060
1061     # fallback: try to get IP by other means
1062     return PVE::Network::get_ip_from_hostname($nodename, $noerr);
1063 }
1064
1065 sub get_local_migration_ip {
1066     my ($migration_network, $noerr) = @_;
1067
1068     my $cidr = $migration_network;
1069
1070     if (!defined($cidr)) {
1071         my $dc_conf = cfs_read_file('datacenter.cfg');
1072         $cidr = $dc_conf->{migration}->{network}
1073           if defined($dc_conf->{migration}->{network});
1074     }
1075
1076     if (defined($cidr)) {
1077         my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
1078
1079         die "could not get migration ip: no IP address configured on local " .
1080             "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
1081
1082         die "could not get migration ip: multiple IP address configured for " .
1083             "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
1084
1085         return @$ips[0];
1086     }
1087
1088     return undef;
1089 };
1090
1091 # ssh related utility functions
1092
1093 sub ssh_merge_keys {
1094     # remove duplicate keys in $sshauthkeys
1095     # ssh-copy-id simply add keys, so the file can grow to large
1096
1097     my $data = '';
1098     if (-f $sshauthkeys) {
1099         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
1100         chomp($data);
1101     }
1102
1103     my $found_backup;
1104     if (-f $rootsshauthkeysbackup) {
1105         $data .= "\n";
1106         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
1107         chomp($data);
1108         $found_backup = 1;
1109     }
1110
1111     # always add ourself
1112     if (-f $ssh_rsa_id) {
1113         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
1114         chomp($pub);
1115         $data .= "\n$pub\n";
1116     }
1117
1118     my $newdata = "";
1119     my $vhash = {};
1120     my @lines = split(/\n/, $data);
1121     foreach my $line (@lines) {
1122         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1123             next if $vhash->{$3}++;
1124         }
1125         $newdata .= "$line\n";
1126     }
1127
1128     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
1129
1130     if ($found_backup && -l $rootsshauthkeys) {
1131         # everything went well, so we can remove the backup
1132         unlink $rootsshauthkeysbackup;
1133     }
1134 }
1135
1136 sub setup_sshd_config {
1137     my () = @_;
1138
1139     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
1140
1141     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1142
1143     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1144         chomp $conf;
1145         $conf .= "\nPermitRootLogin yes\n";
1146     }
1147
1148     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
1149
1150     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
1151 }
1152
1153 sub setup_rootsshconfig {
1154
1155     # create ssh key if it does not exist
1156     if (! -f $ssh_rsa_id) {
1157         mkdir '/root/.ssh/';
1158         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1159     }
1160
1161     # create ssh config if it does not exist
1162     if (! -f $rootsshconfig) {
1163         mkdir '/root/.ssh';
1164         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
1165             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
1166             # changed order to put AES before Chacha20 (most hardware has AESNI)
1167             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
1168             close($fh);
1169         }
1170     }
1171 }
1172
1173 sub setup_ssh_keys {
1174
1175     mkdir $authdir;
1176
1177     my $import_ok;
1178
1179     if (! -f $sshauthkeys) {
1180         my $old;
1181         if (-f $rootsshauthkeys) {
1182             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
1183         }
1184         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1185             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
1186             close($fh);
1187             $import_ok = 1;
1188         }
1189     }
1190
1191     warn "can't create shared ssh key database '$sshauthkeys'\n"
1192         if ! -f $sshauthkeys;
1193
1194     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
1195         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1196             warn "rename $rootsshauthkeys failed - $!\n";
1197         }
1198     }
1199
1200     if (! -l $rootsshauthkeys) {
1201         symlink $sshauthkeys, $rootsshauthkeys;
1202     }
1203
1204     if (! -l $rootsshauthkeys) {
1205         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1206     } else {
1207         unlink $rootsshauthkeysbackup if $import_ok;
1208     }
1209 }
1210
1211 sub ssh_unmerge_known_hosts {
1212     return if ! -l $sshglobalknownhosts;
1213
1214     my $old = '';
1215     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1216         if -f $sshknownhosts;
1217
1218     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1219 }
1220
1221 sub ssh_merge_known_hosts {
1222     my ($nodename, $ip_address, $createLink) = @_;
1223
1224     die "no node name specified" if !$nodename;
1225     die "no ip address specified" if !$ip_address;
1226
1227     # ssh lowercases hostnames (aliases) before comparision, so we need too
1228     $nodename = lc($nodename);
1229     $ip_address = lc($ip_address);
1230
1231     mkdir $authdir;
1232
1233     if (! -f $sshknownhosts) {
1234         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1235             close($fh);
1236         }
1237     }
1238
1239     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1240
1241     my $new = '';
1242
1243     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1244         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1245     }
1246
1247     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1248     # Note: file sometimes containe emty lines at start, so we use multiline match
1249     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
1250     $hostkey = $1;
1251
1252     my $data = '';
1253     my $vhash = {};
1254
1255     my $found_nodename;
1256     my $found_local_ip;
1257
1258     my $merge_line = sub {
1259         my ($line, $all) = @_;
1260
1261         return if $line =~ m/^\s*$/; # skip empty lines
1262         return if $line =~ m/^#/; # skip comments
1263
1264         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1265             my $key = $1;
1266             my $rsakey = $2;
1267             if (!$vhash->{$key}) {
1268                 $vhash->{$key} = 1;
1269                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1270                     my $salt = decode_base64($1);
1271                     my $digest = $2;
1272                     my $hmac = Digest::HMAC_SHA1->new($salt);
1273                     $hmac->add($nodename);
1274                     my $hd = $hmac->b64digest . '=';
1275                     if ($digest eq $hd) {
1276                         if ($rsakey eq $hostkey) {
1277                             $found_nodename = 1;
1278                             $data .= $line;
1279                         }
1280                         return;
1281                     }
1282                     $hmac = Digest::HMAC_SHA1->new($salt);
1283                     $hmac->add($ip_address);
1284                     $hd = $hmac->b64digest . '=';
1285                     if ($digest eq $hd) {
1286                         if ($rsakey eq $hostkey) {
1287                             $found_local_ip = 1;
1288                             $data .= $line;
1289                         }
1290                         return;
1291                     }
1292                 } else {
1293                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
1294                     if ($key eq $ip_address) {
1295                         $found_local_ip = 1 if $rsakey eq $hostkey;
1296                     } elsif ($key eq $nodename) {
1297                         $found_nodename = 1 if $rsakey eq $hostkey;
1298                     }
1299                 }
1300                 $data .= $line;
1301             }
1302         } elsif ($all) {
1303             $data .= $line;
1304         }
1305     };
1306
1307     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1308         my $line = "$2\n";
1309         &$merge_line($line, 1);
1310     }
1311
1312     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1313         my $line = "$2\n";
1314         &$merge_line($line);
1315     }
1316
1317     # add our own key if not already there
1318     $data .= "$nodename $hostkey\n" if !$found_nodename;
1319     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
1320
1321     PVE::Tools::file_set_contents($sshknownhosts, $data);
1322
1323     return if !$createLink;
1324
1325     unlink $sshglobalknownhosts;
1326     symlink $sshknownhosts, $sshglobalknownhosts;
1327
1328     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1329         if ! -l $sshglobalknownhosts;
1330
1331 }
1332
1333 my $migration_format = {
1334     type => {
1335         default_key => 1,
1336         type => 'string',
1337         enum => ['secure', 'insecure'],
1338         description => "Migration traffic is encrypted using an SSH tunnel by " .
1339           "default. On secure, completely private networks this can be " .
1340           "disabled to increase performance.",
1341         default => 'secure',
1342     },
1343     network => {
1344         optional => 1,
1345         type => 'string', format => 'CIDR',
1346         format_description => 'CIDR',
1347         description => "CIDR of the (sub) network that is used for migration."
1348     },
1349 };
1350
1351 my $ha_format = {
1352     shutdown_policy => {
1353         type => 'string',
1354         enum => ['freeze', 'failover', 'conditional'],
1355         description => "The policy for HA services on node shutdown. 'freeze' disables auto-recovery, 'failover' ensures recovery, 'conditional' recovers on poweroff and freezes on reboot. Running HA Services will always get stopped first on shutdown.",
1356         verbose_description => "Describes the policy for handling HA services on poweroff or reboot of a node. Freeze will always freeze services which are still located on the node on shutdown, those services won't be recovered by the HA manager. Failover will not mark the services as frozen and thus the services will get recovered to other nodes, if the shutdown node does not come up again quickly (< 1min). 'conditional' chooses automatically depending on the type of shutdown, i.e., on a reboot the service will be frozen but on a poweroff the service will stay as is, and thus get recovered after about 2 minutes.",
1357         default => 'conditional',
1358     }
1359 };
1360
1361 PVE::JSONSchema::register_format('mac-prefix', \&pve_verify_mac_prefix);
1362 sub pve_verify_mac_prefix {
1363     my ($mac_prefix, $noerr) = @_;
1364
1365     if ($mac_prefix !~ m/^[a-f0-9][02468ace](?::[a-f0-9]{2}){0,2}:?$/i) {
1366         return undef if $noerr;
1367         die "value is not a valid unicast MAC address prefix\n";
1368     }
1369     return $mac_prefix;
1370 }
1371
1372 my $datacenter_schema = {
1373     type => "object",
1374     additionalProperties => 0,
1375     properties => {
1376         keyboard => {
1377             optional => 1,
1378             type => 'string',
1379             description => "Default keybord layout for vnc server.",
1380             enum => PVE::Tools::kvmkeymaplist(),
1381         },
1382         language => {
1383             optional => 1,
1384             type => 'string',
1385             description => "Default GUI language.",
1386             enum => [
1387                 'zh_CN',
1388                 'zh_TW',
1389                 'ca',
1390                 'en',
1391                 'eu',
1392                 'fr',
1393                 'de',
1394                 'it',
1395                 'es',
1396                 'ja',
1397                 'nb',
1398                 'nn',
1399                 'fa',
1400                 'pl',
1401                 'pt_BR',
1402                 'ru',
1403                 'sl',
1404                 'sv',
1405                 'tr',
1406             ],
1407         },
1408         http_proxy => {
1409             optional => 1,
1410             type => 'string',
1411             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1412             pattern => "http://.*",
1413         },
1414         migration_unsecure => {
1415             optional => 1,
1416             type => 'boolean',
1417             description => "Migration is secure using SSH tunnel by default. " .
1418               "For secure private networks you can disable it to speed up " .
1419               "migration. Deprecated, use the 'migration' property instead!",
1420         },
1421         migration => {
1422             optional => 1,
1423             type => 'string', format => $migration_format,
1424             description => "For cluster wide migration settings.",
1425         },
1426         console => {
1427             optional => 1,
1428             type => 'string',
1429             description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), an HTML5 based vnc viewer (noVNC), or an HTML5 based console client (xtermjs). If the selected viewer is not available (e.g. SPICE not activated for the VM), the fallback is noVNC.",
1430             enum => ['applet', 'vv', 'html5', 'xtermjs'],
1431         },
1432         email_from => {
1433             optional => 1,
1434             type => 'string',
1435             format => 'email-opt',
1436             description => "Specify email address to send notification from (default is root@\$hostname)",
1437         },
1438         max_workers => {
1439             optional => 1,
1440             type => 'integer',
1441             minimum => 1,
1442             description => "Defines how many workers (per node) are maximal started ".
1443               " on actions like 'stopall VMs' or task from the ha-manager.",
1444         },
1445         fencing => {
1446             optional => 1,
1447             type => 'string',
1448             default => 'watchdog',
1449             enum => [ 'watchdog', 'hardware', 'both' ],
1450             description => "Set the fencing mode of the HA cluster. Hardware mode " .
1451               "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
1452               " With both all two modes are used." .
1453               "\n\nWARNING: 'hardware' and 'both' are EXPERIMENTAL & WIP",
1454         },
1455         ha => {
1456             optional => 1,
1457             type => 'string', format => $ha_format,
1458             description => "Cluster wide HA settings.",
1459         },
1460         mac_prefix => {
1461             optional => 1,
1462             type => 'string',
1463             format => 'mac-prefix',
1464             description => 'Prefix for autogenerated MAC addresses.',
1465         },
1466         bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
1467     },
1468 };
1469
1470 # make schema accessible from outside (for documentation)
1471 sub get_datacenter_schema { return $datacenter_schema };
1472
1473 sub parse_datacenter_config {
1474     my ($filename, $raw) = @_;
1475
1476     my $res = PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
1477
1478     if (my $migration = $res->{migration}) {
1479         $res->{migration} = PVE::JSONSchema::parse_property_string($migration_format, $migration);
1480     }
1481
1482     if (my $ha = $res->{ha}) {
1483         $res->{ha} = PVE::JSONSchema::parse_property_string($ha_format, $ha);
1484     }
1485
1486     # for backwards compatibility only, new migration property has precedence
1487     if (defined($res->{migration_unsecure})) {
1488         if (defined($res->{migration}->{type})) {
1489             warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
1490               "set at same time! Ignore 'migration_unsecure'\n";
1491         } else {
1492             $res->{migration}->{type} = ($res->{migration_unsecure}) ? 'insecure' : 'secure';
1493         }
1494     }
1495
1496     # for backwards compatibility only, applet maps to html5
1497     if (defined($res->{console}) && $res->{console} eq 'applet') {
1498         $res->{console} = 'html5';
1499     }
1500
1501     return $res;
1502 }
1503
1504 sub write_datacenter_config {
1505     my ($filename, $cfg) = @_;
1506
1507     # map deprecated setting to new one
1508     if (defined($cfg->{migration_unsecure}) && !defined($cfg->{migration})) {
1509         my $migration_unsecure = delete $cfg->{migration_unsecure};
1510         $cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
1511     }
1512
1513     # map deprecated applet setting to html5
1514     if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
1515         $cfg->{console} = 'html5';
1516     }
1517
1518     if (my $migration = $cfg->{migration}) {
1519         $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
1520     }
1521
1522     if (my $ha = $cfg->{ha}) {
1523         $cfg->{ha} = PVE::JSONSchema::print_property_string($ha, $ha_format);
1524     }
1525
1526     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1527 }
1528
1529 cfs_register_file('datacenter.cfg',
1530                   \&parse_datacenter_config,
1531                   \&write_datacenter_config);
1532
1533 # X509 Certificate cache helper
1534
1535 my $cert_cache_nodes = {};
1536 my $cert_cache_timestamp = time();
1537 my $cert_cache_fingerprints = {};
1538
1539 sub update_cert_cache {
1540     my ($update_node, $clear) = @_;
1541
1542     syslog('info', "Clearing outdated entries from certificate cache")
1543         if $clear;
1544
1545     $cert_cache_timestamp = time() if !defined($update_node);
1546
1547     my $node_list = defined($update_node) ?
1548         [ $update_node ] : [ keys %$cert_cache_nodes ];
1549
1550     foreach my $node (@$node_list) {
1551         my $clear_old = sub {
1552             if (my $old_fp = $cert_cache_nodes->{$node}) {
1553                 # distrust old fingerprint
1554                 delete $cert_cache_fingerprints->{$old_fp};
1555                 # ensure reload on next proxied request
1556                 delete $cert_cache_nodes->{$node};
1557             }
1558         };
1559
1560         my $fp = eval { get_node_fingerprint($node) };
1561         if (my $err = $@) {
1562             warn "$err\n";
1563             &$clear_old() if $clear;
1564             next;
1565         }
1566
1567         my $old_fp = $cert_cache_nodes->{$node};
1568         $cert_cache_fingerprints->{$fp} = 1;
1569         $cert_cache_nodes->{$node} = $fp;
1570
1571         if (defined($old_fp) && $fp ne $old_fp) {
1572             delete $cert_cache_fingerprints->{$old_fp};
1573         }
1574     }
1575 }
1576
1577 # load and cache cert fingerprint once
1578 sub initialize_cert_cache {
1579     my ($node) = @_;
1580
1581     update_cert_cache($node)
1582         if defined($node) && !defined($cert_cache_nodes->{$node});
1583 }
1584
1585 sub read_ssl_cert_fingerprint {
1586     my ($cert_path) = @_;
1587
1588     my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
1589         or die "unable to read '$cert_path' - $!\n";
1590
1591     my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
1592     Net::SSLeay::BIO_free($bio);
1593
1594     die "unable to read certificate from '$cert_path'\n" if !$cert;
1595
1596     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1597     Net::SSLeay::X509_free($cert);
1598
1599     die "unable to get fingerprint for '$cert_path' - got empty value\n"
1600         if !defined($fp) || $fp eq '';
1601
1602     return $fp;
1603 }
1604
1605 sub get_node_fingerprint {
1606     my ($node) = @_;
1607
1608     my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
1609     my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
1610
1611     $cert_path = $custom_cert_path if -f $custom_cert_path;
1612
1613     return read_ssl_cert_fingerprint($cert_path);
1614 }
1615
1616
1617 sub check_cert_fingerprint {
1618     my ($cert) = @_;
1619
1620     # clear cache every 30 minutes at least
1621     update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
1622
1623     # get fingerprint of server certificate
1624     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1625     return 0 if !defined($fp) || $fp eq ''; # error
1626
1627     my $check = sub {
1628         for my $expected (keys %$cert_cache_fingerprints) {
1629             return 1 if $fp eq $expected;
1630         }
1631         return 0;
1632     };
1633
1634     return 1 if &$check();
1635
1636     # clear cache and retry at most once every minute
1637     if (time() - $cert_cache_timestamp >= 60) {
1638         syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
1639         update_cert_cache();
1640         return &$check();
1641     }
1642
1643     return 0;
1644 }
1645
1646 # bash completion helpers
1647
1648 sub complete_next_vmid {
1649
1650     my $vmlist = get_vmlist() || {};
1651     my $idlist = $vmlist->{ids} || {};
1652
1653     for (my $i = 100; $i < 10000; $i++) {
1654         return [$i] if !defined($idlist->{$i});
1655     }
1656
1657     return [];
1658 }
1659
1660 sub complete_vmid {
1661
1662     my $vmlist = get_vmlist();
1663     my $ids = $vmlist->{ids} || {};
1664
1665     return [ keys %$ids ];
1666 }
1667
1668 sub complete_local_vmid {
1669
1670     my $vmlist = get_vmlist();
1671     my $ids = $vmlist->{ids} || {};
1672
1673     my $nodename = PVE::INotify::nodename();
1674
1675     my $res = [];
1676     foreach my $vmid (keys %$ids) {
1677         my $d = $ids->{$vmid};
1678         next if !$d->{node} || $d->{node} ne $nodename;
1679         push @$res, $vmid;
1680     }
1681
1682     return $res;
1683 }
1684
1685 sub complete_migration_target {
1686
1687     my $res = [];
1688
1689     my $nodename = PVE::INotify::nodename();
1690
1691     my $nodelist = get_nodelist();
1692     foreach my $node (@$nodelist) {
1693         next if $node eq $nodename;
1694         push @$res, $node;
1695     }
1696
1697     return $res;
1698 }
1699
1700 sub get_ssh_info {
1701     my ($node, $network_cidr) = @_;
1702
1703     my $ip;
1704     if (defined($network_cidr)) {
1705         # Use mtunnel via to get the remote node's ip inside $network_cidr.
1706         # This goes over the regular network (iow. uses get_ssh_info() with
1707         # $network_cidr undefined.
1708         # FIXME: Use the REST API client for this after creating an API entry
1709         # for get_migration_ip.
1710         my $default_remote = get_ssh_info($node, undef);
1711         my $default_ssh = ssh_info_to_command($default_remote);
1712         my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
1713             '-migration_network', $network_cidr,
1714             '-get_migration_ip'
1715         ];
1716         PVE::Tools::run_command($cmd, outfunc => sub {
1717             my ($line) = @_;
1718             chomp $line;
1719             die "internal error: unexpected output from mtunnel\n"
1720                 if defined($ip);
1721             if ($line =~ /^ip: '(.*)'$/) {
1722                 $ip = $1;
1723             } else {
1724                 die "internal error: bad output from mtunnel\n"
1725                     if defined($ip);
1726             }
1727         });
1728         die "failed to get ip for node '$node' in network '$network_cidr'\n"
1729             if !defined($ip);
1730     } else {
1731         $ip = remote_node_ip($node);
1732     }
1733
1734     return {
1735         ip => $ip,
1736         name => $node,
1737         network => $network_cidr,
1738     };
1739 }
1740
1741 sub ssh_info_to_command_base {
1742     my ($info, @extra_options) = @_;
1743     return [
1744         '/usr/bin/ssh',
1745         '-e', 'none',
1746         '-o', 'BatchMode=yes',
1747         '-o', 'HostKeyAlias='.$info->{name},
1748         @extra_options
1749     ];
1750 }
1751
1752 sub ssh_info_to_command {
1753     my ($info, @extra_options) = @_;
1754     my $cmd = ssh_info_to_command_base($info, @extra_options);
1755     push @$cmd, "root\@$info->{ip}";
1756     return $cmd;
1757 }
1758
1759 sub assert_joinable {
1760     my ($ring0_addr, $ring1_addr, $force) = @_;
1761
1762     my $errors = '';
1763     my $error = sub { $errors .= "* $_[0]\n"; };
1764
1765     if (-f $authfile) {
1766         $error->("authentication key '$authfile' already exists");
1767     }
1768
1769     if (-f $clusterconf)  {
1770         $error->("cluster config '$clusterconf' already exists");
1771     }
1772
1773     my $vmlist = get_vmlist();
1774     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
1775         $error->("this host already contains virtual guests");
1776     }
1777
1778     if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
1779         $error->("corosync is already running, is this node already in a cluster?!");
1780     }
1781
1782     # check if corosync ring IPs are configured on the current nodes interfaces
1783     my $check_ip = sub {
1784         my $ip = shift // return;
1785         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
1786             my $host = $ip;
1787             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
1788             if ($@) {
1789                 $error->("cannot use '$host': $@\n") ;
1790                 return;
1791             }
1792         }
1793
1794         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
1795         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
1796
1797         $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n")
1798             if (scalar(@$configured_ips) != 1);
1799     };
1800
1801     $check_ip->($ring0_addr);
1802     $check_ip->($ring1_addr);
1803
1804     if ($errors) {
1805         warn "detected the following error(s):\n$errors";
1806         die "Check if node may join a cluster failed!\n" if !$force;
1807     }
1808 }
1809
1810 # NOTE: filesystem must be offline here, no DB changes allowed
1811 my $backup_cfs_database = sub {
1812     my ($dbfile) = @_;
1813
1814     mkdir $dbbackupdir;
1815
1816     my $ctime = time();
1817     my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
1818
1819     print "backup old database to '$backup_fn'\n";
1820
1821     my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \ ">${backup_fn}"] ];
1822     run_command($cmd, 'errmsg' => "cannot backup old database\n");
1823
1824     my $maxfiles = 10; # purge older backup
1825     my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql.gz> ];
1826
1827     if ((my $count = scalar(@$backups)) > $maxfiles) {
1828         foreach my $f (@$backups[$maxfiles..$count-1]) {
1829             next if $f !~ m/^(\S+)$/; # untaint
1830             print "delete old backup '$1'\n";
1831             unlink $1;
1832         }
1833     }
1834 };
1835
1836 sub join {
1837     my ($param) = @_;
1838
1839     my $nodename = PVE::INotify::nodename();
1840
1841     setup_sshd_config();
1842     setup_rootsshconfig();
1843     setup_ssh_keys();
1844
1845     # check if we can join with the given parameters and current node state
1846     my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
1847     assert_joinable($ring0_addr, $ring1_addr, $param->{force});
1848
1849     # make sure known_hosts is on local filesystem
1850     ssh_unmerge_known_hosts();
1851
1852     my $host = $param->{hostname};
1853     my $local_ip_address = remote_node_ip($nodename);
1854
1855     my $conn_args = {
1856         username => 'root@pam',
1857         password => $param->{password},
1858         cookie_name => 'PVEAuthCookie',
1859         protocol => 'https',
1860         host => $host,
1861         port => 8006,
1862     };
1863
1864     if (my $fp = $param->{fingerprint}) {
1865         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
1866     } else {
1867         # API schema ensures that we can only get here from CLI handler
1868         $conn_args->{manual_verification} = 1;
1869     }
1870
1871     print "Establishing API connection with host '$host'\n";
1872
1873     my $conn = PVE::APIClient::LWP->new(%$conn_args);
1874     $conn->login();
1875
1876     # login raises an exception on failure, so if we get here we're good
1877     print "Login succeeded.\n";
1878
1879     my $args = {};
1880     $args->{force} = $param->{force} if defined($param->{force});
1881     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
1882     $args->{votes} = $param->{votes} if defined($param->{votes});
1883     $args->{ring0_addr} = $ring0_addr // $local_ip_address;
1884     $args->{ring1_addr} = $ring1_addr if defined($ring1_addr);
1885
1886     print "Request addition of this node\n";
1887     my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
1888
1889     print "Join request OK, finishing setup locally\n";
1890
1891     # added successfuly - now prepare local node
1892     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
1893 }
1894
1895 sub finish_join {
1896     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
1897
1898     mkdir "$localclusterdir";
1899     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
1900     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
1901
1902     print "stopping pve-cluster service\n";
1903     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
1904     run_command($cmd, errmsg => "can't stop pve-cluster service");
1905
1906     $backup_cfs_database->($dbfile);
1907     unlink $dbfile;
1908
1909     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
1910     run_command($cmd, errmsg => "starting pve-cluster failed");
1911
1912     # wait for quorum
1913     my $printqmsg = 1;
1914     while (!check_cfs_quorum(1)) {
1915         if ($printqmsg) {
1916             print "waiting for quorum...";
1917             STDOUT->flush();
1918             $printqmsg = 0;
1919         }
1920         sleep(1);
1921     }
1922     print "OK\n" if !$printqmsg;
1923
1924     updatecerts_and_ssh(1);
1925
1926     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
1927     run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
1928
1929     print "successfully added node '$nodename' to cluster.\n";
1930 }
1931
1932 sub updatecerts_and_ssh {
1933     my ($force_new_cert, $silent) = @_;
1934
1935     my $p = sub { print "$_[0]\n" if !$silent };
1936
1937     setup_rootsshconfig();
1938
1939     gen_pve_vzdump_symlink();
1940
1941     if (!check_cfs_quorum(1)) {
1942         return undef if $silent;
1943         die "no quorum - unable to update files\n";
1944     }
1945
1946     setup_ssh_keys();
1947
1948     my $nodename = PVE::INotify::nodename();
1949     my $local_ip_address = remote_node_ip($nodename);
1950
1951     $p->("(re)generate node files");
1952     $p->("generate new node certificate") if $force_new_cert;
1953     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
1954
1955     $p->("merge authorized SSH keys and known hosts");
1956     ssh_merge_keys();
1957     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
1958     gen_pve_vzdump_files();
1959 }
1960
1961 1;