data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EEXIST ENOENT);
   6 use File::stat qw();
   7 use Socket;
   8 use Storable qw(dclone);
   9 use IO::File;
  10 use MIME::Base64;
  11 use Digest::SHA;
  12 use Digest::HMAC_SHA1;
  13 use Net::SSLeay;
  14 use PVE::Tools qw(run_command);
  15 use PVE::INotify;
  16 use PVE::IPCC;
  17 use PVE::SafeSyslog;
  18 use PVE::JSONSchema;
  19 use PVE::Network;
  20 use PVE::Cluster::IPCConst;
  21 use JSON;
  22 use RRDs;
  23 use Encode;
  24 use UUID;
  25 use base 'Exporter';
  26
  27 our @EXPORT_OK = qw(
  28 cfs_read_file
  29 cfs_write_file
  30 cfs_register_file
  31 cfs_lock_file);
  32
  33 use Data::Dumper; # fixme: remove
  34
  35 # x509 certificate utils
  36
  37 my $basedir = "/etc/pve";
  38 my $authdir = "$basedir/priv";
  39 my $lockdir = "/etc/pve/priv/lock";
  40
  41 # cfs and corosync files
  42 my $dbfile = "/var/lib/pve-cluster/config.db";
  43 my $dbbackupdir = "/var/lib/pve-cluster/backup";
  44 my $localclusterdir = "/etc/corosync";
  45 my $localclusterconf = "$localclusterdir/corosync.conf";
  46 my $authfile = "$localclusterdir/authkey";
  47 my $clusterconf = "$basedir/corosync.conf";
  48
  49 my $authprivkeyfn = "$authdir/authkey.key";
  50 my $authpubkeyfn = "$basedir/authkey.pub";
  51 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  52 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  53 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  54 # this is just a secret accessable by the web browser
  55 # and is used for CSRF prevention
  56 my $pvewww_key_fn = "$basedir/pve-www.key";
  57
  58 # ssh related files
  59 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  60 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  61 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  62 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  63 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  64 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  65 my $sshd_config_fn = "/etc/ssh/sshd_config";
  66 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  67 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  68 my $rootsshconfig = "/root/.ssh/config";
  69
  70 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
  71 # observed files are the one we can get directly through IPCC, they are cached
  72 # using a computed version and only those can be used by the cfs_*_file methods
  73 my $observed = {
  74     'vzdump.cron' => 1,
  75     'storage.cfg' => 1,
  76     'datacenter.cfg' => 1,
  77     'replication.cfg' => 1,
  78     'corosync.conf' => 1,
  79     'corosync.conf.new' => 1,
  80     'user.cfg' => 1,
  81     'domains.cfg' => 1,
  82     'priv/shadow.cfg' => 1,
  83     'priv/tfa.cfg' => 1,
  84     '/qemu-server/' => 1,
  85     '/openvz/' => 1,
  86     '/lxc/' => 1,
  87     'ha/crm_commands' => 1,
  88     'ha/manager_status' => 1,
  89     'ha/resources.cfg' => 1,
  90     'ha/groups.cfg' => 1,
  91     'ha/fence.cfg' => 1,
  92     'status.cfg' => 1,
  93     'ceph.conf' => 1,
  94 };
  95
  96 # only write output if something fails
  97 sub run_silent_cmd {
  98     my ($cmd) = @_;
  99
 100     my $outbuf = '';
 101     my $record = sub { $outbuf .= shift . "\n"; };
 102
 103     eval { run_command($cmd, outfunc => $record, errfunc => $record) };
 104
 105     if (my $err = $@) {
 106         print STDERR $outbuf;
 107         die $err;
 108     }
 109 }
 110
 111 sub check_cfs_quorum {
 112     my ($noerr) = @_;
 113
 114     # note: -w filename always return 1 for root, so wee need
 115     # to use File::lstat here
 116     my $st = File::stat::lstat("$basedir/local");
 117     my $quorate = ($st && (($st->mode & 0200) != 0));
 118
 119     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 120
 121     return $quorate;
 122 }
 123
 124 sub check_cfs_is_mounted {
 125     my ($noerr) = @_;
 126
 127     my $res = -l "$basedir/local";
 128
 129     die "pve configuration filesystem not mounted\n"
 130         if !$res && !$noerr;
 131
 132     return $res;
 133 }
 134
 135 sub gen_local_dirs {
 136     my ($nodename) = @_;
 137
 138     check_cfs_is_mounted();
 139
 140     my @required_dirs = (
 141         "$basedir/priv",
 142         "$basedir/nodes",
 143         "$basedir/nodes/$nodename",
 144         "$basedir/nodes/$nodename/lxc",
 145         "$basedir/nodes/$nodename/qemu-server",
 146         "$basedir/nodes/$nodename/openvz",
 147         "$basedir/nodes/$nodename/priv");
 148
 149     foreach my $dir (@required_dirs) {
 150         if (! -d $dir) {
 151             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 152         }
 153     }
 154 }
 155
 156 sub gen_auth_key {
 157
 158     return if -f "$authprivkeyfn";
 159
 160     check_cfs_is_mounted();
 161
 162     cfs_lock_authkey(undef, sub {
 163         mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
 164
 165         run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 166
 167         run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 168     });
 169
 170     die "$@\n" if $@;
 171 }
 172
 173 sub gen_pveca_key {
 174
 175     return if -f $pveca_key_fn;
 176
 177     eval {
 178         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 179     };
 180
 181     die "unable to generate pve ca key:\n$@" if $@;
 182 }
 183
 184 sub gen_pveca_cert {
 185
 186     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 187         return 0;
 188     }
 189
 190     gen_pveca_key();
 191
 192     # we try to generate an unique 'subject' to avoid browser problems
 193     # (reused serial numbers, ..)
 194     my $uuid;
 195     UUID::generate($uuid);
 196     my $uuid_str;
 197     UUID::unparse($uuid, $uuid_str);
 198
 199     eval {
 200         # wrap openssl with faketime to prevent bug #904
 201         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 202                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 203                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 204                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 205     };
 206
 207     die "generating pve root certificate failed:\n$@" if $@;
 208
 209     return 1;
 210 }
 211
 212 sub gen_pve_ssl_key {
 213     my ($nodename) = @_;
 214
 215     die "no node name specified" if !$nodename;
 216
 217     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 218
 219     return if -f $pvessl_key_fn;
 220
 221     eval {
 222         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 223     };
 224
 225     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 226 }
 227
 228 sub gen_pve_www_key {
 229
 230     return if -f $pvewww_key_fn;
 231
 232     eval {
 233         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 234     };
 235
 236     die "unable to generate pve www key:\n$@" if $@;
 237 }
 238
 239 sub update_serial {
 240     my ($serial) = @_;
 241
 242     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 243 }
 244
 245 sub gen_pve_ssl_cert {
 246     my ($force, $nodename, $ip) = @_;
 247
 248     die "no node name specified" if !$nodename;
 249     die "no IP specified" if !$ip;
 250
 251     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 252
 253     return if !$force && -f $pvessl_cert_fn;
 254
 255     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 256
 257     my $rc = PVE::INotify::read_file('resolvconf');
 258
 259     $names .= ",IP:$ip";
 260
 261     my $fqdn = $nodename;
 262
 263     $names .= ",DNS:$nodename";
 264
 265     if ($rc && $rc->{search}) {
 266         $fqdn = $nodename . "." . $rc->{search};
 267         $names .= ",DNS:$fqdn";
 268     }
 269
 270     my $sslconf = <<__EOD;
 271 RANDFILE = /root/.rnd
 272 extensions = v3_req
 273
 274 [ req ]
 275 default_bits = 2048
 276 distinguished_name = req_distinguished_name
 277 req_extensions = v3_req
 278 prompt = no
 279 string_mask = nombstr
 280
 281 [ req_distinguished_name ]
 282 organizationalUnitName = PVE Cluster Node
 283 organizationName = Proxmox Virtual Environment
 284 commonName = $fqdn
 285
 286 [ v3_req ]
 287 basicConstraints = CA:FALSE
 288 extendedKeyUsage = serverAuth
 289 subjectAltName = $names
 290 __EOD
 291
 292     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 293     my $fh = IO::File->new ($cfgfn, "w");
 294     print $fh $sslconf;
 295     close ($fh);
 296
 297     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 298     unlink $reqfn;
 299
 300     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 301     eval {
 302         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 303                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 304     };
 305
 306     if (my $err = $@) {
 307         unlink $reqfn;
 308         unlink $cfgfn;
 309         die "unable to generate pve certificate request:\n$err";
 310     }
 311
 312     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 313
 314     eval {
 315         # wrap openssl with faketime to prevent bug #904
 316         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
 317                         '-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
 318                         '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
 319                         '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
 320     };
 321
 322     if (my $err = $@) {
 323         unlink $reqfn;
 324         unlink $cfgfn;
 325         die "unable to generate pve ssl certificate:\n$err";
 326     }
 327
 328     unlink $cfgfn;
 329     unlink $reqfn;
 330 }
 331
 332 sub gen_pve_node_files {
 333     my ($nodename, $ip, $opt_force) = @_;
 334
 335     gen_local_dirs($nodename);
 336
 337     gen_auth_key();
 338
 339     # make sure we have a (cluster wide) secret
 340     # for CSRFR prevention
 341     gen_pve_www_key();
 342
 343     # make sure we have a (per node) private key
 344     gen_pve_ssl_key($nodename);
 345
 346     # make sure we have a CA
 347     my $force = gen_pveca_cert();
 348
 349     $force = 1 if $opt_force;
 350
 351     gen_pve_ssl_cert($force, $nodename, $ip);
 352 }
 353
 354 my $vzdump_cron_dummy = <<__EOD;
 355 # cluster wide vzdump cron schedule
 356 # Atomatically generated file - do not edit
 357
 358 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 359
 360 __EOD
 361
 362 sub gen_pve_vzdump_symlink {
 363
 364     my $filename = "/etc/pve/vzdump.cron";
 365
 366     my $link_fn = "/etc/cron.d/vzdump";
 367
 368     if ((-f $filename) && (! -l $link_fn)) {
 369         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 370         symlink($filename, $link_fn);
 371     }
 372 }
 373
 374 sub gen_pve_vzdump_files {
 375
 376     my $filename = "/etc/pve/vzdump.cron";
 377
 378     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 379         if ! -f $filename;
 380
 381     gen_pve_vzdump_symlink();
 382 };
 383
 384 my $versions = {};
 385 my $vmlist = {};
 386 my $clinfo = {};
 387
 388 my $ipcc_send_rec = sub {
 389     my ($msgid, $data) = @_;
 390
 391     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 392
 393     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 394
 395     return $res;
 396 };
 397
 398 my $ipcc_send_rec_json = sub {
 399     my ($msgid, $data) = @_;
 400
 401     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 402
 403     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 404
 405     return decode_json($res);
 406 };
 407
 408 my $ipcc_get_config = sub {
 409     my ($path) = @_;
 410
 411     my $bindata = pack "Z*", $path;
 412     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_CONFIG, $bindata);
 413     if (!defined($res)) {
 414         if ($! != 0) {
 415             return undef if $! == ENOENT;
 416             die "$!\n";
 417         }
 418         return '';
 419     }
 420
 421     return $res;
 422 };
 423
 424 my $ipcc_get_status = sub {
 425     my ($name, $nodename) = @_;
 426
 427     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 428     return PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_STATUS, $bindata);
 429 };
 430
 431 my $ipcc_update_status = sub {
 432     my ($name, $data) = @_;
 433
 434     my $raw = ref($data) ? encode_json($data) : $data;
 435     # update status
 436     my $bindata = pack "Z[256]Z*", $name, $raw;
 437
 438     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 439 };
 440
 441 my $ipcc_log = sub {
 442     my ($priority, $ident, $tag, $msg) = @_;
 443
 444     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 445     bytes::length($tag) + 1, $ident, $tag, $msg;
 446
 447     return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG, $bindata);
 448 };
 449
 450 my $ipcc_get_cluster_log = sub {
 451     my ($user, $max) = @_;
 452
 453     $max = 0 if !defined($max);
 454
 455     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 456     return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG, $bindata);
 457 };
 458
 459 my $ccache = {};
 460
 461 sub cfs_update {
 462     my ($fail) = @_;
 463     eval {
 464         my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION);
 465         #warn "GOT1: " . Dumper($res);
 466         die "no starttime\n" if !$res->{starttime};
 467
 468         if (!$res->{starttime} || !$versions->{starttime} ||
 469             $res->{starttime} != $versions->{starttime}) {
 470             #print "detected changed starttime\n";
 471             $vmlist = {};
 472             $clinfo = {};
 473             $ccache = {};
 474         }
 475
 476         $versions = $res;
 477     };
 478     my $err = $@;
 479     if ($err) {
 480         $versions = {};
 481         $vmlist = {};
 482         $clinfo = {};
 483         $ccache = {};
 484         die $err if $fail;
 485         warn $err;
 486     }
 487
 488     eval {
 489         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 490             #warn "detected new clinfo\n";
 491             $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO);
 492         }
 493     };
 494     $err = $@;
 495     if ($err) {
 496         $clinfo = {};
 497         die $err if $fail;
 498         warn $err;
 499     }
 500
 501     eval {
 502         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 503             #warn "detected new vmlist1\n";
 504             $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST);
 505         }
 506     };
 507     $err = $@;
 508     if ($err) {
 509         $vmlist = {};
 510         die $err if $fail;
 511         warn $err;
 512     }
 513 }
 514
 515 sub get_vmlist {
 516     return $vmlist;
 517 }
 518
 519 sub get_clinfo {
 520     return $clinfo;
 521 }
 522
 523 sub get_members {
 524     return $clinfo->{nodelist};
 525 }
 526
 527 sub get_nodelist {
 528     my $nodelist = $clinfo->{nodelist};
 529
 530     my $nodename = PVE::INotify::nodename();
 531
 532     if (!$nodelist || !$nodelist->{$nodename}) {
 533         return [ $nodename ];
 534     }
 535
 536     return [ keys %$nodelist ];
 537 }
 538
 539 # $data must be a chronological descending ordered array of tasks
 540 sub broadcast_tasklist {
 541     my ($data) = @_;
 542
 543     # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
 544     # from pmxcfs) - drop older items until we satisfy this constraint
 545     my $size = length(encode_json($data));
 546     while ($size >= (32 * 1024)) {
 547         pop @$data;
 548         $size = length(encode_json($data));
 549     }
 550
 551     eval {
 552         &$ipcc_update_status("tasklist", $data);
 553     };
 554
 555     warn $@ if $@;
 556 }
 557
 558 my $tasklistcache = {};
 559
 560 sub get_tasklist {
 561     my ($nodename) = @_;
 562
 563     my $kvstore = $versions->{kvstore} || {};
 564
 565     my $nodelist = get_nodelist();
 566
 567     my $res = [];
 568     foreach my $node (@$nodelist) {
 569         next if $nodename && ($nodename ne $node);
 570         eval {
 571             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 572             my $cd = $tasklistcache->{$node};
 573             if (!$cd || !$ver || !$cd->{version} ||
 574                 ($cd->{version} != $ver)) {
 575                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 576                 my $data = decode_json($raw);
 577                 push @$res, @$data;
 578                 $cd = $tasklistcache->{$node} = {
 579                     data => $data,
 580                     version => $ver,
 581                 };
 582             } elsif ($cd && $cd->{data}) {
 583                 push @$res, @{$cd->{data}};
 584             }
 585         };
 586         my $err = $@;
 587         syslog('err', $err) if $err;
 588     }
 589
 590     return $res;
 591 }
 592
 593 sub broadcast_rrd {
 594     my ($rrdid, $data) = @_;
 595
 596     eval {
 597         &$ipcc_update_status("rrd/$rrdid", $data);
 598     };
 599     my $err = $@;
 600
 601     warn $err if $err;
 602 }
 603
 604 my $last_rrd_dump = 0;
 605 my $last_rrd_data = "";
 606
 607 sub rrd_dump {
 608
 609     my $ctime = time();
 610
 611     my $diff = $ctime - $last_rrd_dump;
 612     if ($diff < 2) {
 613         return $last_rrd_data;
 614     }
 615
 616     my $raw;
 617     eval {
 618         $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP);
 619     };
 620     my $err = $@;
 621
 622     if ($err) {
 623         warn $err;
 624         return {};
 625     }
 626
 627     my $res = {};
 628
 629     if ($raw) {
 630         while ($raw =~ s/^(.*)\n//) {
 631             my ($key, @ela) = split(/:/, $1);
 632             next if !$key;
 633             next if !(scalar(@ela) > 1);
 634             $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
 635         }
 636     }
 637
 638     $last_rrd_dump = $ctime;
 639     $last_rrd_data = $res;
 640
 641     return $res;
 642 }
 643
 644 sub create_rrd_data {
 645     my ($rrdname, $timeframe, $cf) = @_;
 646
 647     my $rrddir = "/var/lib/rrdcached/db";
 648
 649     my $rrd = "$rrddir/$rrdname";
 650
 651     my $setup = {
 652         hour =>  [ 60, 70 ],
 653         day  =>  [ 60*30, 70 ],
 654         week =>  [ 60*180, 70 ],
 655         month => [ 60*720, 70 ],
 656         year =>  [ 60*10080, 70 ],
 657     };
 658
 659     my ($reso, $count) = @{$setup->{$timeframe}};
 660     my $ctime  = $reso*int(time()/$reso);
 661     my $req_start = $ctime - $reso*$count;
 662
 663     $cf = "AVERAGE" if !$cf;
 664
 665     my @args = (
 666         "-s" => $req_start,
 667         "-e" => $ctime - 1,
 668         "-r" => $reso,
 669         );
 670
 671     my $socket = "/var/run/rrdcached.sock";
 672     push @args, "--daemon" => "unix:$socket" if -S $socket;
 673
 674     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 675
 676     my $err = RRDs::error;
 677     die "RRD error: $err\n" if $err;
 678
 679     die "got wrong time resolution ($step != $reso)\n"
 680         if $step != $reso;
 681
 682     my $res = [];
 683     my $fields = scalar(@$names);
 684     for my $line (@$data) {
 685         my $entry = { 'time' => $start };
 686         $start += $step;
 687         for (my $i = 0; $i < $fields; $i++) {
 688             my $name = $names->[$i];
 689             if (defined(my $val = $line->[$i])) {
 690                 $entry->{$name} = $val;
 691             } else {
 692                 # leave empty fields undefined
 693                 # maybe make this configurable?
 694             }
 695         }
 696         push @$res, $entry;
 697     }
 698
 699     return $res;
 700 }
 701
 702 sub create_rrd_graph {
 703     my ($rrdname, $timeframe, $ds, $cf) = @_;
 704
 705     # Using RRD graph is clumsy - maybe it
 706     # is better to simply fetch the data, and do all display
 707     # related things with javascript (new extjs html5 graph library).
 708
 709     my $rrddir = "/var/lib/rrdcached/db";
 710
 711     my $rrd = "$rrddir/$rrdname";
 712
 713     my @ids = PVE::Tools::split_list($ds);
 714
 715     my $ds_txt = join('_', @ids);
 716
 717     my $filename = "${rrd}_${ds_txt}.png";
 718
 719     my $setup = {
 720         hour =>  [ 60, 60 ],
 721         day  =>  [ 60*30, 70 ],
 722         week =>  [ 60*180, 70 ],
 723         month => [ 60*720, 70 ],
 724         year =>  [ 60*10080, 70 ],
 725     };
 726
 727     my ($reso, $count) = @{$setup->{$timeframe}};
 728
 729     my @args = (
 730         "--imgformat" => "PNG",
 731         "--border" => 0,
 732         "--height" => 200,
 733         "--width" => 800,
 734         "--start" => - $reso*$count,
 735         "--end" => 'now' ,
 736         "--lower-limit" => 0,
 737         );
 738
 739     my $socket = "/var/run/rrdcached.sock";
 740     push @args, "--daemon" => "unix:$socket" if -S $socket;
 741
 742     my @coldef = ('#00ddff', '#ff0000');
 743
 744     $cf = "AVERAGE" if !$cf;
 745
 746     my $i = 0;
 747     foreach my $id (@ids) {
 748         my $col = $coldef[$i++] || die "fixme: no color definition";
 749         push @args, "DEF:${id}=$rrd:${id}:$cf";
 750         my $dataid = $id;
 751         if ($id eq 'cpu' || $id eq 'iowait') {
 752             push @args, "CDEF:${id}_per=${id},100,*";
 753             $dataid = "${id}_per";
 754         }
 755         push @args, "LINE2:${dataid}${col}:${id}";
 756     }
 757
 758     push @args, '--full-size-mode';
 759
 760     # we do not really store data into the file
 761     my $res = RRDs::graphv('-', @args);
 762
 763     my $err = RRDs::error;
 764     die "RRD error: $err\n" if $err;
 765
 766     return { filename => $filename, image => $res->{image} };
 767 }
 768
 769 # a fast way to read files (avoid fuse overhead)
 770 sub get_config {
 771     my ($path) = @_;
 772
 773     return &$ipcc_get_config($path);
 774 }
 775
 776 sub get_cluster_log {
 777     my ($user, $max) = @_;
 778
 779     return &$ipcc_get_cluster_log($user, $max);
 780 }
 781
 782 my $file_info = {};
 783
 784 sub cfs_register_file {
 785     my ($filename, $parser, $writer) = @_;
 786
 787     $observed->{$filename} || die "unknown file '$filename'";
 788
 789     die "file '$filename' already registered" if $file_info->{$filename};
 790
 791     $file_info->{$filename} = {
 792         parser => $parser,
 793         writer => $writer,
 794     };
 795 }
 796
 797 my $ccache_read = sub {
 798     my ($filename, $parser, $version) = @_;
 799
 800     $ccache->{$filename} = {} if !$ccache->{$filename};
 801
 802     my $ci = $ccache->{$filename};
 803
 804     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 805         # we always call the parser, even when the file does not exists
 806         # (in that case $data is undef)
 807         my $data = get_config($filename);
 808         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 809         $ci->{version} = $version;
 810     }
 811
 812     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 813
 814     return $res;
 815 };
 816
 817 sub cfs_file_version {
 818     my ($filename) = @_;
 819
 820     my $version;
 821     my $infotag;
 822     if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
 823         my ($type, $vmid) = ($1, $2);
 824         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 825             $version = $vmlist->{ids}->{$vmid}->{version};
 826         }
 827         $infotag = "/$type/";
 828     } else {
 829         $infotag = $filename;
 830         $version = $versions->{$filename};
 831     }
 832
 833     my $info = $file_info->{$infotag} ||
 834         die "unknown file type '$filename'\n";
 835
 836     return wantarray ? ($version, $info) : $version;
 837 }
 838
 839 sub cfs_read_file {
 840     my ($filename) = @_;
 841
 842     my ($version, $info) = cfs_file_version($filename);
 843     my $parser = $info->{parser};
 844
 845     return &$ccache_read($filename, $parser, $version);
 846 }
 847
 848 sub cfs_write_file {
 849     my ($filename, $data) = @_;
 850
 851     my ($version, $info) = cfs_file_version($filename);
 852
 853     my $writer = $info->{writer} || die "no writer defined";
 854
 855     my $fsname = "/etc/pve/$filename";
 856
 857     my $raw = &$writer($fsname, $data);
 858
 859     if (my $ci = $ccache->{$filename}) {
 860         $ci->{version} = undef;
 861     }
 862
 863     PVE::Tools::file_set_contents($fsname, $raw);
 864 }
 865
 866 my $cfs_lock = sub {
 867     my ($lockid, $timeout, $code, @param) = @_;
 868
 869     my $prev_alarm = alarm(0); # suspend outer alarm early
 870
 871     my $res;
 872     my $got_lock = 0;
 873
 874     # this timeout is for acquire the lock
 875     $timeout = 10 if !$timeout;
 876
 877     my $filename = "$lockdir/$lockid";
 878
 879     eval {
 880
 881         mkdir $lockdir;
 882
 883         if (! -d $lockdir) {
 884             die "pve cluster filesystem not online.\n";
 885         }
 886
 887         my $timeout_err = sub { die "got lock request timeout\n"; };
 888         local $SIG{ALRM} = $timeout_err;
 889
 890         while (1) {
 891             alarm ($timeout);
 892             $got_lock = mkdir($filename);
 893             $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
 894
 895             last if $got_lock;
 896
 897             $timeout_err->() if $timeout <= 0;
 898
 899             print STDERR "trying to acquire cfs lock '$lockid' ...\n";
 900             utime (0, 0, $filename); # cfs unlock request
 901             sleep(1);
 902         }
 903
 904         # fixed command timeout: cfs locks have a timeout of 120
 905         # using 60 gives us another 60 seconds to abort the task
 906         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 907         alarm(60);
 908
 909         cfs_update(); # make sure we read latest versions inside code()
 910
 911         $res = &$code(@param);
 912
 913         alarm(0);
 914     };
 915
 916     my $err = $@;
 917
 918     $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
 919
 920     rmdir $filename if $got_lock; # if we held the lock always unlock again
 921
 922     alarm($prev_alarm);
 923
 924     if ($err) {
 925         $@ = "error with cfs lock '$lockid': $err";
 926         return undef;
 927     }
 928
 929     $@ = undef;
 930
 931     return $res;
 932 };
 933
 934 sub cfs_lock_file {
 935     my ($filename, $timeout, $code, @param) = @_;
 936
 937     my $info = $observed->{$filename} || die "unknown file '$filename'";
 938
 939     my $lockid = "file-$filename";
 940     $lockid =~ s/[.\/]/_/g;
 941
 942     &$cfs_lock($lockid, $timeout, $code, @param);
 943 }
 944
 945 sub cfs_lock_storage {
 946     my ($storeid, $timeout, $code, @param) = @_;
 947
 948     my $lockid = "storage-$storeid";
 949
 950     &$cfs_lock($lockid, $timeout, $code, @param);
 951 }
 952
 953 sub cfs_lock_domain {
 954     my ($domainname, $timeout, $code, @param) = @_;
 955
 956     my $lockid = "domain-$domainname";
 957
 958     &$cfs_lock($lockid, $timeout, $code, @param);
 959 }
 960
 961 sub cfs_lock_acme {
 962     my ($account, $timeout, $code, @param) = @_;
 963
 964     my $lockid = "acme-$account";
 965
 966     &$cfs_lock($lockid, $timeout, $code, @param);
 967 }
 968
 969 sub cfs_lock_authkey {
 970     my ($timeout, $code, @param) = @_;
 971
 972     $cfs_lock->('authkey', $timeout, $code, @param);
 973 }
 974
 975 my $log_levels = {
 976     "emerg" => 0,
 977     "alert" => 1,
 978     "crit" => 2,
 979     "critical" => 2,
 980     "err" => 3,
 981     "error" => 3,
 982     "warn" => 4,
 983     "warning" => 4,
 984     "notice" => 5,
 985     "info" => 6,
 986     "debug" => 7,
 987 };
 988
 989 sub log_msg {
 990    my ($priority, $ident, $msg) = @_;
 991
 992    if (my $tmp = $log_levels->{$priority}) {
 993        $priority = $tmp;
 994    }
 995
 996    die "need numeric log priority" if $priority !~ /^\d+$/;
 997
 998    my $tag = PVE::SafeSyslog::tag();
 999
1000    $msg = "empty message" if !$msg;
1001
1002    $ident = "" if !$ident;
1003    $ident = encode("ascii", $ident,
1004                    sub { sprintf "\\u%04x", shift });
1005
1006    my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
1007
1008    if ($ident) {
1009        syslog($priority, "<%s> %s", $ident, $ascii);
1010    } else {
1011        syslog($priority, "%s", $ascii);
1012    }
1013
1014    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
1015
1016    syslog("err", "writing cluster log failed: $@") if $@;
1017 }
1018
1019 sub check_vmid_unused {
1020     my ($vmid, $noerr) = @_;
1021
1022     my $vmlist = get_vmlist();
1023
1024     my $d = $vmlist->{ids}->{$vmid};
1025     return 1 if !defined($d);
1026
1027     return undef if $noerr;
1028
1029     my $vmtypestr =  $d->{type} eq 'qemu' ? 'VM' : 'CT';
1030     die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
1031 }
1032
1033 sub check_node_exists {
1034     my ($nodename, $noerr) = @_;
1035
1036     my $nodelist = $clinfo->{nodelist};
1037     return 1 if $nodelist && $nodelist->{$nodename};
1038
1039     return undef if $noerr;
1040
1041     die "no such cluster node '$nodename'\n";
1042 }
1043
1044 # this is also used to get the IP of the local node
1045 sub remote_node_ip {
1046     my ($nodename, $noerr) = @_;
1047
1048     my $nodelist = $clinfo->{nodelist};
1049     if ($nodelist && $nodelist->{$nodename}) {
1050         if (my $ip = $nodelist->{$nodename}->{ip}) {
1051             return $ip if !wantarray;
1052             my $family = $nodelist->{$nodename}->{address_family};
1053             if (!$family) {
1054                 $nodelist->{$nodename}->{address_family} =
1055                     $family =
1056                     PVE::Tools::get_host_address_family($ip);
1057             }
1058             return wantarray ? ($ip, $family) : $ip;
1059         }
1060     }
1061
1062     # fallback: try to get IP by other means
1063     return PVE::Network::get_ip_from_hostname($nodename, $noerr);
1064 }
1065
1066 sub get_local_migration_ip {
1067     my ($migration_network, $noerr) = @_;
1068
1069     my $cidr = $migration_network;
1070
1071     if (!defined($cidr)) {
1072         my $dc_conf = cfs_read_file('datacenter.cfg');
1073         $cidr = $dc_conf->{migration}->{network}
1074           if defined($dc_conf->{migration}->{network});
1075     }
1076
1077     if (defined($cidr)) {
1078         my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
1079
1080         die "could not get migration ip: no IP address configured on local " .
1081             "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
1082
1083         die "could not get migration ip: multiple IP address configured for " .
1084             "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
1085
1086         return @$ips[0];
1087     }
1088
1089     return undef;
1090 };
1091
1092 # ssh related utility functions
1093
1094 sub ssh_merge_keys {
1095     # remove duplicate keys in $sshauthkeys
1096     # ssh-copy-id simply add keys, so the file can grow to large
1097
1098     my $data = '';
1099     if (-f $sshauthkeys) {
1100         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
1101         chomp($data);
1102     }
1103
1104     my $found_backup;
1105     if (-f $rootsshauthkeysbackup) {
1106         $data .= "\n";
1107         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
1108         chomp($data);
1109         $found_backup = 1;
1110     }
1111
1112     # always add ourself
1113     if (-f $ssh_rsa_id) {
1114         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
1115         chomp($pub);
1116         $data .= "\n$pub\n";
1117     }
1118
1119     my $newdata = "";
1120     my $vhash = {};
1121     my @lines = split(/\n/, $data);
1122     foreach my $line (@lines) {
1123         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1124             next if $vhash->{$3}++;
1125         }
1126         $newdata .= "$line\n";
1127     }
1128
1129     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
1130
1131     if ($found_backup && -l $rootsshauthkeys) {
1132         # everything went well, so we can remove the backup
1133         unlink $rootsshauthkeysbackup;
1134     }
1135 }
1136
1137 sub setup_sshd_config {
1138     my () = @_;
1139
1140     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
1141
1142     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1143
1144     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1145         chomp $conf;
1146         $conf .= "\nPermitRootLogin yes\n";
1147     }
1148
1149     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
1150
1151     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
1152 }
1153
1154 sub setup_rootsshconfig {
1155
1156     # create ssh key if it does not exist
1157     if (! -f $ssh_rsa_id) {
1158         mkdir '/root/.ssh/';
1159         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1160     }
1161
1162     # create ssh config if it does not exist
1163     if (! -f $rootsshconfig) {
1164         mkdir '/root/.ssh';
1165         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
1166             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
1167             # changed order to put AES before Chacha20 (most hardware has AESNI)
1168             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
1169             close($fh);
1170         }
1171     }
1172 }
1173
1174 sub setup_ssh_keys {
1175
1176     mkdir $authdir;
1177
1178     my $import_ok;
1179
1180     if (! -f $sshauthkeys) {
1181         my $old;
1182         if (-f $rootsshauthkeys) {
1183             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
1184         }
1185         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1186             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
1187             close($fh);
1188             $import_ok = 1;
1189         }
1190     }
1191
1192     warn "can't create shared ssh key database '$sshauthkeys'\n"
1193         if ! -f $sshauthkeys;
1194
1195     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
1196         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1197             warn "rename $rootsshauthkeys failed - $!\n";
1198         }
1199     }
1200
1201     if (! -l $rootsshauthkeys) {
1202         symlink $sshauthkeys, $rootsshauthkeys;
1203     }
1204
1205     if (! -l $rootsshauthkeys) {
1206         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1207     } else {
1208         unlink $rootsshauthkeysbackup if $import_ok;
1209     }
1210 }
1211
1212 sub ssh_unmerge_known_hosts {
1213     return if ! -l $sshglobalknownhosts;
1214
1215     my $old = '';
1216     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1217         if -f $sshknownhosts;
1218
1219     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1220 }
1221
1222 sub ssh_merge_known_hosts {
1223     my ($nodename, $ip_address, $createLink) = @_;
1224
1225     die "no node name specified" if !$nodename;
1226     die "no ip address specified" if !$ip_address;
1227
1228     # ssh lowercases hostnames (aliases) before comparision, so we need too
1229     $nodename = lc($nodename);
1230     $ip_address = lc($ip_address);
1231
1232     mkdir $authdir;
1233
1234     if (! -f $sshknownhosts) {
1235         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1236             close($fh);
1237         }
1238     }
1239
1240     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1241
1242     my $new = '';
1243
1244     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1245         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1246     }
1247
1248     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1249     # Note: file sometimes containe emty lines at start, so we use multiline match
1250     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
1251     $hostkey = $1;
1252
1253     my $data = '';
1254     my $vhash = {};
1255
1256     my $found_nodename;
1257     my $found_local_ip;
1258
1259     my $merge_line = sub {
1260         my ($line, $all) = @_;
1261
1262         return if $line =~ m/^\s*$/; # skip empty lines
1263         return if $line =~ m/^#/; # skip comments
1264
1265         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1266             my $key = $1;
1267             my $rsakey = $2;
1268             if (!$vhash->{$key}) {
1269                 $vhash->{$key} = 1;
1270                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1271                     my $salt = decode_base64($1);
1272                     my $digest = $2;
1273                     my $hmac = Digest::HMAC_SHA1->new($salt);
1274                     $hmac->add($nodename);
1275                     my $hd = $hmac->b64digest . '=';
1276                     if ($digest eq $hd) {
1277                         if ($rsakey eq $hostkey) {
1278                             $found_nodename = 1;
1279                             $data .= $line;
1280                         }
1281                         return;
1282                     }
1283                     $hmac = Digest::HMAC_SHA1->new($salt);
1284                     $hmac->add($ip_address);
1285                     $hd = $hmac->b64digest . '=';
1286                     if ($digest eq $hd) {
1287                         if ($rsakey eq $hostkey) {
1288                             $found_local_ip = 1;
1289                             $data .= $line;
1290                         }
1291                         return;
1292                     }
1293                 } else {
1294                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
1295                     if ($key eq $ip_address) {
1296                         $found_local_ip = 1 if $rsakey eq $hostkey;
1297                     } elsif ($key eq $nodename) {
1298                         $found_nodename = 1 if $rsakey eq $hostkey;
1299                     }
1300                 }
1301                 $data .= $line;
1302             }
1303         } elsif ($all) {
1304             $data .= $line;
1305         }
1306     };
1307
1308     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1309         my $line = "$2\n";
1310         &$merge_line($line, 1);
1311     }
1312
1313     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1314         my $line = "$2\n";
1315         &$merge_line($line);
1316     }
1317
1318     # add our own key if not already there
1319     $data .= "$nodename $hostkey\n" if !$found_nodename;
1320     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
1321
1322     PVE::Tools::file_set_contents($sshknownhosts, $data);
1323
1324     return if !$createLink;
1325
1326     unlink $sshglobalknownhosts;
1327     symlink $sshknownhosts, $sshglobalknownhosts;
1328
1329     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1330         if ! -l $sshglobalknownhosts;
1331
1332 }
1333
1334 my $migration_format = {
1335     type => {
1336         default_key => 1,
1337         type => 'string',
1338         enum => ['secure', 'insecure'],
1339         description => "Migration traffic is encrypted using an SSH tunnel by " .
1340           "default. On secure, completely private networks this can be " .
1341           "disabled to increase performance.",
1342         default => 'secure',
1343     },
1344     network => {
1345         optional => 1,
1346         type => 'string', format => 'CIDR',
1347         format_description => 'CIDR',
1348         description => "CIDR of the (sub) network that is used for migration."
1349     },
1350 };
1351
1352 my $ha_format = {
1353     shutdown_policy => {
1354         type => 'string',
1355         enum => ['freeze', 'failover', 'conditional'],
1356         description => "The policy for HA services on node shutdown. 'freeze' disables auto-recovery, 'failover' ensures recovery, 'conditional' recovers on poweroff and freezes on reboot. Running HA Services will always get stopped first on shutdown.",
1357         verbose_description => "Describes the policy for handling HA services on poweroff or reboot of a node. Freeze will always freeze services which are still located on the node on shutdown, those services won't be recovered by the HA manager. Failover will not mark the services as frozen and thus the services will get recovered to other nodes, if the shutdown node does not come up again quickly (< 1min). 'conditional' chooses automatically depending on the type of shutdown, i.e., on a reboot the service will be frozen but on a poweroff the service will stay as is, and thus get recovered after about 2 minutes.",
1358         default => 'conditional',
1359     }
1360 };
1361
1362 PVE::JSONSchema::register_format('mac-prefix', \&pve_verify_mac_prefix);
1363 sub pve_verify_mac_prefix {
1364     my ($mac_prefix, $noerr) = @_;
1365
1366     if ($mac_prefix !~ m/^[a-f0-9][02468ace](?::[a-f0-9]{2}){0,2}:?$/i) {
1367         return undef if $noerr;
1368         die "value is not a valid unicast MAC address prefix\n";
1369     }
1370     return $mac_prefix;
1371 }
1372
1373 our $u2f_format = {
1374     appid => {
1375         type => 'string',
1376         description => "U2F AppId URL override. Defaults to the origin.",
1377         format_description => 'APPID',
1378         optional => 1,
1379     },
1380     origin => {
1381         type => 'string',
1382         description => "U2F Origin override. Mostly useful for single nodes with a single URL.",
1383         format_description => 'URL',
1384         optional => 1,
1385     },
1386 };
1387
1388 my $datacenter_schema = {
1389     type => "object",
1390     additionalProperties => 0,
1391     properties => {
1392         keyboard => {
1393             optional => 1,
1394             type => 'string',
1395             description => "Default keybord layout for vnc server.",
1396             enum => PVE::Tools::kvmkeymaplist(),
1397         },
1398         language => {
1399             optional => 1,
1400             type => 'string',
1401             description => "Default GUI language.",
1402             enum => [
1403                 'zh_CN',
1404                 'zh_TW',
1405                 'ca',
1406                 'en',
1407                 'eu',
1408                 'fr',
1409                 'de',
1410                 'it',
1411                 'es',
1412                 'ja',
1413                 'nb',
1414                 'nn',
1415                 'fa',
1416                 'pl',
1417                 'pt_BR',
1418                 'ru',
1419                 'sl',
1420                 'sv',
1421                 'tr',
1422             ],
1423         },
1424         http_proxy => {
1425             optional => 1,
1426             type => 'string',
1427             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1428             pattern => "http://.*",
1429         },
1430         migration_unsecure => {
1431             optional => 1,
1432             type => 'boolean',
1433             description => "Migration is secure using SSH tunnel by default. " .
1434               "For secure private networks you can disable it to speed up " .
1435               "migration. Deprecated, use the 'migration' property instead!",
1436         },
1437         migration => {
1438             optional => 1,
1439             type => 'string', format => $migration_format,
1440             description => "For cluster wide migration settings.",
1441         },
1442         console => {
1443             optional => 1,
1444             type => 'string',
1445             description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), an HTML5 based vnc viewer (noVNC), or an HTML5 based console client (xtermjs). If the selected viewer is not available (e.g. SPICE not activated for the VM), the fallback is noVNC.",
1446             enum => ['applet', 'vv', 'html5', 'xtermjs'],
1447         },
1448         email_from => {
1449             optional => 1,
1450             type => 'string',
1451             format => 'email-opt',
1452             description => "Specify email address to send notification from (default is root@\$hostname)",
1453         },
1454         max_workers => {
1455             optional => 1,
1456             type => 'integer',
1457             minimum => 1,
1458             description => "Defines how many workers (per node) are maximal started ".
1459               " on actions like 'stopall VMs' or task from the ha-manager.",
1460         },
1461         fencing => {
1462             optional => 1,
1463             type => 'string',
1464             default => 'watchdog',
1465             enum => [ 'watchdog', 'hardware', 'both' ],
1466             description => "Set the fencing mode of the HA cluster. Hardware mode " .
1467               "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
1468               " With both all two modes are used." .
1469               "\n\nWARNING: 'hardware' and 'both' are EXPERIMENTAL & WIP",
1470         },
1471         ha => {
1472             optional => 1,
1473             type => 'string', format => $ha_format,
1474             description => "Cluster wide HA settings.",
1475         },
1476         mac_prefix => {
1477             optional => 1,
1478             type => 'string',
1479             format => 'mac-prefix',
1480             description => 'Prefix for autogenerated MAC addresses.',
1481         },
1482         bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
1483         u2f => {
1484             optional => 1,
1485             type => 'string',
1486             format => $u2f_format,
1487             description => 'u2f',
1488         },
1489     },
1490 };
1491
1492 # make schema accessible from outside (for documentation)
1493 sub get_datacenter_schema { return $datacenter_schema };
1494
1495 sub parse_datacenter_config {
1496     my ($filename, $raw) = @_;
1497
1498     my $res = PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
1499
1500     if (my $migration = $res->{migration}) {
1501         $res->{migration} = PVE::JSONSchema::parse_property_string($migration_format, $migration);
1502     }
1503
1504     if (my $ha = $res->{ha}) {
1505         $res->{ha} = PVE::JSONSchema::parse_property_string($ha_format, $ha);
1506     }
1507
1508     # for backwards compatibility only, new migration property has precedence
1509     if (defined($res->{migration_unsecure})) {
1510         if (defined($res->{migration}->{type})) {
1511             warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
1512               "set at same time! Ignore 'migration_unsecure'\n";
1513         } else {
1514             $res->{migration}->{type} = ($res->{migration_unsecure}) ? 'insecure' : 'secure';
1515         }
1516     }
1517
1518     # for backwards compatibility only, applet maps to html5
1519     if (defined($res->{console}) && $res->{console} eq 'applet') {
1520         $res->{console} = 'html5';
1521     }
1522
1523     return $res;
1524 }
1525
1526 sub write_datacenter_config {
1527     my ($filename, $cfg) = @_;
1528
1529     # map deprecated setting to new one
1530     if (defined($cfg->{migration_unsecure}) && !defined($cfg->{migration})) {
1531         my $migration_unsecure = delete $cfg->{migration_unsecure};
1532         $cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
1533     }
1534
1535     # map deprecated applet setting to html5
1536     if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
1537         $cfg->{console} = 'html5';
1538     }
1539
1540     if (ref($cfg->{migration})) {
1541         my $migration = $cfg->{migration};
1542         $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
1543     }
1544
1545     if (ref($cfg->{ha})) {
1546         my $ha = $cfg->{ha};
1547         $cfg->{ha} = PVE::JSONSchema::print_property_string($ha, $ha_format);
1548     }
1549
1550     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1551 }
1552
1553 cfs_register_file('datacenter.cfg',
1554                   \&parse_datacenter_config,
1555                   \&write_datacenter_config);
1556
1557 # X509 Certificate cache helper
1558
1559 my $cert_cache_nodes = {};
1560 my $cert_cache_timestamp = time();
1561 my $cert_cache_fingerprints = {};
1562
1563 sub update_cert_cache {
1564     my ($update_node, $clear) = @_;
1565
1566     syslog('info', "Clearing outdated entries from certificate cache")
1567         if $clear;
1568
1569     $cert_cache_timestamp = time() if !defined($update_node);
1570
1571     my $node_list = defined($update_node) ?
1572         [ $update_node ] : [ keys %$cert_cache_nodes ];
1573
1574     foreach my $node (@$node_list) {
1575         my $clear_old = sub {
1576             if (my $old_fp = $cert_cache_nodes->{$node}) {
1577                 # distrust old fingerprint
1578                 delete $cert_cache_fingerprints->{$old_fp};
1579                 # ensure reload on next proxied request
1580                 delete $cert_cache_nodes->{$node};
1581             }
1582         };
1583
1584         my $fp = eval { get_node_fingerprint($node) };
1585         if (my $err = $@) {
1586             warn "$err\n";
1587             &$clear_old() if $clear;
1588             next;
1589         }
1590
1591         my $old_fp = $cert_cache_nodes->{$node};
1592         $cert_cache_fingerprints->{$fp} = 1;
1593         $cert_cache_nodes->{$node} = $fp;
1594
1595         if (defined($old_fp) && $fp ne $old_fp) {
1596             delete $cert_cache_fingerprints->{$old_fp};
1597         }
1598     }
1599 }
1600
1601 # load and cache cert fingerprint once
1602 sub initialize_cert_cache {
1603     my ($node) = @_;
1604
1605     update_cert_cache($node)
1606         if defined($node) && !defined($cert_cache_nodes->{$node});
1607 }
1608
1609 sub read_ssl_cert_fingerprint {
1610     my ($cert_path) = @_;
1611
1612     my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
1613         or die "unable to read '$cert_path' - $!\n";
1614
1615     my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
1616     Net::SSLeay::BIO_free($bio);
1617
1618     die "unable to read certificate from '$cert_path'\n" if !$cert;
1619
1620     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1621     Net::SSLeay::X509_free($cert);
1622
1623     die "unable to get fingerprint for '$cert_path' - got empty value\n"
1624         if !defined($fp) || $fp eq '';
1625
1626     return $fp;
1627 }
1628
1629 sub get_node_fingerprint {
1630     my ($node) = @_;
1631
1632     my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
1633     my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
1634
1635     $cert_path = $custom_cert_path if -f $custom_cert_path;
1636
1637     return read_ssl_cert_fingerprint($cert_path);
1638 }
1639
1640
1641 sub check_cert_fingerprint {
1642     my ($cert) = @_;
1643
1644     # clear cache every 30 minutes at least
1645     update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
1646
1647     # get fingerprint of server certificate
1648     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1649     return 0 if !defined($fp) || $fp eq ''; # error
1650
1651     my $check = sub {
1652         for my $expected (keys %$cert_cache_fingerprints) {
1653             return 1 if $fp eq $expected;
1654         }
1655         return 0;
1656     };
1657
1658     return 1 if &$check();
1659
1660     # clear cache and retry at most once every minute
1661     if (time() - $cert_cache_timestamp >= 60) {
1662         syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
1663         update_cert_cache();
1664         return &$check();
1665     }
1666
1667     return 0;
1668 }
1669
1670 # bash completion helpers
1671
1672 sub complete_next_vmid {
1673
1674     my $vmlist = get_vmlist() || {};
1675     my $idlist = $vmlist->{ids} || {};
1676
1677     for (my $i = 100; $i < 10000; $i++) {
1678         return [$i] if !defined($idlist->{$i});
1679     }
1680
1681     return [];
1682 }
1683
1684 sub complete_vmid {
1685
1686     my $vmlist = get_vmlist();
1687     my $ids = $vmlist->{ids} || {};
1688
1689     return [ keys %$ids ];
1690 }
1691
1692 sub complete_local_vmid {
1693
1694     my $vmlist = get_vmlist();
1695     my $ids = $vmlist->{ids} || {};
1696
1697     my $nodename = PVE::INotify::nodename();
1698
1699     my $res = [];
1700     foreach my $vmid (keys %$ids) {
1701         my $d = $ids->{$vmid};
1702         next if !$d->{node} || $d->{node} ne $nodename;
1703         push @$res, $vmid;
1704     }
1705
1706     return $res;
1707 }
1708
1709 sub complete_migration_target {
1710
1711     my $res = [];
1712
1713     my $nodename = PVE::INotify::nodename();
1714
1715     my $nodelist = get_nodelist();
1716     foreach my $node (@$nodelist) {
1717         next if $node eq $nodename;
1718         push @$res, $node;
1719     }
1720
1721     return $res;
1722 }
1723
1724 sub get_ssh_info {
1725     my ($node, $network_cidr) = @_;
1726
1727     my $ip;
1728     if (defined($network_cidr)) {
1729         # Use mtunnel via to get the remote node's ip inside $network_cidr.
1730         # This goes over the regular network (iow. uses get_ssh_info() with
1731         # $network_cidr undefined.
1732         # FIXME: Use the REST API client for this after creating an API entry
1733         # for get_migration_ip.
1734         my $default_remote = get_ssh_info($node, undef);
1735         my $default_ssh = ssh_info_to_command($default_remote);
1736         my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
1737             '-migration_network', $network_cidr,
1738             '-get_migration_ip'
1739         ];
1740         PVE::Tools::run_command($cmd, outfunc => sub {
1741             my ($line) = @_;
1742             chomp $line;
1743             die "internal error: unexpected output from mtunnel\n"
1744                 if defined($ip);
1745             if ($line =~ /^ip: '(.*)'$/) {
1746                 $ip = $1;
1747             } else {
1748                 die "internal error: bad output from mtunnel\n"
1749                     if defined($ip);
1750             }
1751         });
1752         die "failed to get ip for node '$node' in network '$network_cidr'\n"
1753             if !defined($ip);
1754     } else {
1755         $ip = remote_node_ip($node);
1756     }
1757
1758     return {
1759         ip => $ip,
1760         name => $node,
1761         network => $network_cidr,
1762     };
1763 }
1764
1765 sub ssh_info_to_command_base {
1766     my ($info, @extra_options) = @_;
1767     return [
1768         '/usr/bin/ssh',
1769         '-e', 'none',
1770         '-o', 'BatchMode=yes',
1771         '-o', 'HostKeyAlias='.$info->{name},
1772         @extra_options
1773     ];
1774 }
1775
1776 sub ssh_info_to_command {
1777     my ($info, @extra_options) = @_;
1778     my $cmd = ssh_info_to_command_base($info, @extra_options);
1779     push @$cmd, "root\@$info->{ip}";
1780     return $cmd;
1781 }
1782
1783 sub assert_joinable {
1784     my ($ring0_addr, $ring1_addr, $force) = @_;
1785
1786     my $errors = '';
1787     my $error = sub { $errors .= "* $_[0]\n"; };
1788
1789     if (-f $authfile) {
1790         $error->("authentication key '$authfile' already exists");
1791     }
1792
1793     if (-f $clusterconf)  {
1794         $error->("cluster config '$clusterconf' already exists");
1795     }
1796
1797     my $vmlist = get_vmlist();
1798     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
1799         $error->("this host already contains virtual guests");
1800     }
1801
1802     if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
1803         $error->("corosync is already running, is this node already in a cluster?!");
1804     }
1805
1806     # check if corosync ring IPs are configured on the current nodes interfaces
1807     my $check_ip = sub {
1808         my $ip = shift // return;
1809         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
1810             my $host = $ip;
1811             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
1812             if ($@) {
1813                 $error->("cannot use '$host': $@\n") ;
1814                 return;
1815             }
1816         }
1817
1818         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
1819         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
1820
1821         $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n")
1822             if (scalar(@$configured_ips) != 1);
1823     };
1824
1825     $check_ip->($ring0_addr);
1826     $check_ip->($ring1_addr);
1827
1828     if ($errors) {
1829         warn "detected the following error(s):\n$errors";
1830         die "Check if node may join a cluster failed!\n" if !$force;
1831     }
1832 }
1833
1834 # NOTE: filesystem must be offline here, no DB changes allowed
1835 my $backup_cfs_database = sub {
1836     my ($dbfile) = @_;
1837
1838     mkdir $dbbackupdir;
1839
1840     my $ctime = time();
1841     my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
1842
1843     print "backup old database to '$backup_fn'\n";
1844
1845     my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \ ">${backup_fn}"] ];
1846     run_command($cmd, 'errmsg' => "cannot backup old database\n");
1847
1848     my $maxfiles = 10; # purge older backup
1849     my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql.gz> ];
1850
1851     if ((my $count = scalar(@$backups)) > $maxfiles) {
1852         foreach my $f (@$backups[$maxfiles..$count-1]) {
1853             next if $f !~ m/^(\S+)$/; # untaint
1854             print "delete old backup '$1'\n";
1855             unlink $1;
1856         }
1857     }
1858 };
1859
1860 sub join {
1861     my ($param) = @_;
1862
1863     my $nodename = PVE::INotify::nodename();
1864
1865     setup_sshd_config();
1866     setup_rootsshconfig();
1867     setup_ssh_keys();
1868
1869     # check if we can join with the given parameters and current node state
1870     my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
1871     assert_joinable($ring0_addr, $ring1_addr, $param->{force});
1872
1873     # make sure known_hosts is on local filesystem
1874     ssh_unmerge_known_hosts();
1875
1876     my $host = $param->{hostname};
1877     my $local_ip_address = remote_node_ip($nodename);
1878
1879     my $conn_args = {
1880         username => 'root@pam',
1881         password => $param->{password},
1882         cookie_name => 'PVEAuthCookie',
1883         protocol => 'https',
1884         host => $host,
1885         port => 8006,
1886     };
1887
1888     if (my $fp = $param->{fingerprint}) {
1889         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
1890     } else {
1891         # API schema ensures that we can only get here from CLI handler
1892         $conn_args->{manual_verification} = 1;
1893     }
1894
1895     print "Establishing API connection with host '$host'\n";
1896
1897     my $conn = PVE::APIClient::LWP->new(%$conn_args);
1898     $conn->login();
1899
1900     # login raises an exception on failure, so if we get here we're good
1901     print "Login succeeded.\n";
1902
1903     my $args = {};
1904     $args->{force} = $param->{force} if defined($param->{force});
1905     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
1906     $args->{votes} = $param->{votes} if defined($param->{votes});
1907     $args->{ring0_addr} = $ring0_addr // $local_ip_address;
1908     $args->{ring1_addr} = $ring1_addr if defined($ring1_addr);
1909
1910     print "Request addition of this node\n";
1911     my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
1912
1913     print "Join request OK, finishing setup locally\n";
1914
1915     # added successfuly - now prepare local node
1916     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
1917 }
1918
1919 sub finish_join {
1920     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
1921
1922     mkdir "$localclusterdir";
1923     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
1924     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
1925
1926     print "stopping pve-cluster service\n";
1927     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
1928     run_command($cmd, errmsg => "can't stop pve-cluster service");
1929
1930     $backup_cfs_database->($dbfile);
1931     unlink $dbfile;
1932
1933     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
1934     run_command($cmd, errmsg => "starting pve-cluster failed");
1935
1936     # wait for quorum
1937     my $printqmsg = 1;
1938     while (!check_cfs_quorum(1)) {
1939         if ($printqmsg) {
1940             print "waiting for quorum...";
1941             STDOUT->flush();
1942             $printqmsg = 0;
1943         }
1944         sleep(1);
1945     }
1946     print "OK\n" if !$printqmsg;
1947
1948     updatecerts_and_ssh(1);
1949
1950     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
1951     run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
1952
1953     print "successfully added node '$nodename' to cluster.\n";
1954 }
1955
1956 sub updatecerts_and_ssh {
1957     my ($force_new_cert, $silent) = @_;
1958
1959     my $p = sub { print "$_[0]\n" if !$silent };
1960
1961     setup_rootsshconfig();
1962
1963     gen_pve_vzdump_symlink();
1964
1965     if (!check_cfs_quorum(1)) {
1966         return undef if $silent;
1967         die "no quorum - unable to update files\n";
1968     }
1969
1970     setup_ssh_keys();
1971
1972     my $nodename = PVE::INotify::nodename();
1973     my $local_ip_address = remote_node_ip($nodename);
1974
1975     $p->("(re)generate node files");
1976     $p->("generate new node certificate") if $force_new_cert;
1977     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
1978
1979     $p->("merge authorized SSH keys and known hosts");
1980     ssh_merge_keys();
1981     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
1982     gen_pve_vzdump_files();
1983 }
1984
1985 1;