data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Digest::HMAC_SHA1;
   7 use Digest::SHA;
   8 use Encode;
   9 use File::stat qw();
  10 use IO::File;
  11 use JSON;
  12 use MIME::Base64;
  13 use Net::SSLeay;
  14 use POSIX qw(EEXIST ENOENT);
  15 use RRDs;
  16 use Socket;
  17 use Storable qw(dclone);
  18 use UUID;
  19
  20 use PVE::INotify;
  21 use PVE::IPCC;
  22 use PVE::JSONSchema;
  23 use PVE::Network;
  24 use PVE::SafeSyslog;
  25 use PVE::Tools qw(run_command);
  26
  27 use PVE::Cluster::IPCConst;
  28
  29 use base 'Exporter';
  30
  31 our @EXPORT_OK = qw(
  32 cfs_read_file
  33 cfs_write_file
  34 cfs_register_file
  35 cfs_lock_file);
  36
  37 # x509 certificate utils
  38
  39 my $basedir = "/etc/pve";
  40 my $authdir = "$basedir/priv";
  41 my $lockdir = "/etc/pve/priv/lock";
  42
  43 # cfs and corosync files
  44 my $dbfile = "/var/lib/pve-cluster/config.db";
  45 my $dbbackupdir = "/var/lib/pve-cluster/backup";
  46 my $localclusterdir = "/etc/corosync";
  47 my $localclusterconf = "$localclusterdir/corosync.conf";
  48 my $authfile = "$localclusterdir/authkey";
  49 my $clusterconf = "$basedir/corosync.conf";
  50
  51 my $authprivkeyfn = "$authdir/authkey.key";
  52 my $authpubkeyfn = "$basedir/authkey.pub";
  53 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  54 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  55 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  56 # this is just a secret accessable by the web browser
  57 # and is used for CSRF prevention
  58 my $pvewww_key_fn = "$basedir/pve-www.key";
  59
  60 # ssh related files
  61 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  62 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  63 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  64 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  65 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  66 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  67 my $sshd_config_fn = "/etc/ssh/sshd_config";
  68 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  69 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  70 my $rootsshconfig = "/root/.ssh/config";
  71
  72 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
  73 # observed files are the one we can get directly through IPCC, they are cached
  74 # using a computed version and only those can be used by the cfs_*_file methods
  75 my $observed = {
  76     'vzdump.cron' => 1,
  77     'storage.cfg' => 1,
  78     'datacenter.cfg' => 1,
  79     'replication.cfg' => 1,
  80     'corosync.conf' => 1,
  81     'corosync.conf.new' => 1,
  82     'user.cfg' => 1,
  83     'domains.cfg' => 1,
  84     'priv/shadow.cfg' => 1,
  85     'priv/tfa.cfg' => 1,
  86     '/qemu-server/' => 1,
  87     '/openvz/' => 1,
  88     '/lxc/' => 1,
  89     'ha/crm_commands' => 1,
  90     'ha/manager_status' => 1,
  91     'ha/resources.cfg' => 1,
  92     'ha/groups.cfg' => 1,
  93     'ha/fence.cfg' => 1,
  94     'status.cfg' => 1,
  95     'ceph.conf' => 1,
  96     'sdn.cfg' => 1,
  97     'sdn.cfg.new' => 1,
  98 };
  99
 100 # only write output if something fails
 101 sub run_silent_cmd {
 102     my ($cmd) = @_;
 103
 104     my $outbuf = '';
 105     my $record = sub { $outbuf .= shift . "\n"; };
 106
 107     eval { run_command($cmd, outfunc => $record, errfunc => $record) };
 108
 109     if (my $err = $@) {
 110         print STDERR $outbuf;
 111         die $err;
 112     }
 113 }
 114
 115 sub check_cfs_quorum {
 116     my ($noerr) = @_;
 117
 118     # note: -w filename always return 1 for root, so wee need
 119     # to use File::lstat here
 120     my $st = File::stat::lstat("$basedir/local");
 121     my $quorate = ($st && (($st->mode & 0200) != 0));
 122
 123     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 124
 125     return $quorate;
 126 }
 127
 128 sub check_cfs_is_mounted {
 129     my ($noerr) = @_;
 130
 131     my $res = -l "$basedir/local";
 132
 133     die "pve configuration filesystem not mounted\n"
 134         if !$res && !$noerr;
 135
 136     return $res;
 137 }
 138
 139 sub gen_local_dirs {
 140     my ($nodename) = @_;
 141
 142     check_cfs_is_mounted();
 143
 144     my @required_dirs = (
 145         "$basedir/priv",
 146         "$basedir/nodes",
 147         "$basedir/nodes/$nodename",
 148         "$basedir/nodes/$nodename/lxc",
 149         "$basedir/nodes/$nodename/qemu-server",
 150         "$basedir/nodes/$nodename/openvz",
 151         "$basedir/nodes/$nodename/priv");
 152
 153     foreach my $dir (@required_dirs) {
 154         if (! -d $dir) {
 155             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 156         }
 157     }
 158 }
 159
 160 sub gen_auth_key {
 161
 162     return if -f "$authprivkeyfn";
 163
 164     check_cfs_is_mounted();
 165
 166     cfs_lock_authkey(undef, sub {
 167         mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
 168
 169         run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 170
 171         run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 172     });
 173
 174     die "$@\n" if $@;
 175 }
 176
 177 sub gen_pveca_key {
 178
 179     return if -f $pveca_key_fn;
 180
 181     eval {
 182         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 183     };
 184
 185     die "unable to generate pve ca key:\n$@" if $@;
 186 }
 187
 188 sub gen_pveca_cert {
 189
 190     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 191         return 0;
 192     }
 193
 194     gen_pveca_key();
 195
 196     # we try to generate an unique 'subject' to avoid browser problems
 197     # (reused serial numbers, ..)
 198     my $uuid;
 199     UUID::generate($uuid);
 200     my $uuid_str;
 201     UUID::unparse($uuid, $uuid_str);
 202
 203     eval {
 204         # wrap openssl with faketime to prevent bug #904
 205         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 206                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 207                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 208                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 209     };
 210
 211     die "generating pve root certificate failed:\n$@" if $@;
 212
 213     return 1;
 214 }
 215
 216 sub gen_pve_ssl_key {
 217     my ($nodename) = @_;
 218
 219     die "no node name specified" if !$nodename;
 220
 221     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 222
 223     return if -f $pvessl_key_fn;
 224
 225     eval {
 226         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 227     };
 228
 229     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 230 }
 231
 232 sub gen_pve_www_key {
 233
 234     return if -f $pvewww_key_fn;
 235
 236     eval {
 237         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 238     };
 239
 240     die "unable to generate pve www key:\n$@" if $@;
 241 }
 242
 243 sub update_serial {
 244     my ($serial) = @_;
 245
 246     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 247 }
 248
 249 sub gen_pve_ssl_cert {
 250     my ($force, $nodename, $ip) = @_;
 251
 252     die "no node name specified" if !$nodename;
 253     die "no IP specified" if !$ip;
 254
 255     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 256
 257     return if !$force && -f $pvessl_cert_fn;
 258
 259     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 260
 261     my $rc = PVE::INotify::read_file('resolvconf');
 262
 263     $names .= ",IP:$ip";
 264
 265     my $fqdn = $nodename;
 266
 267     $names .= ",DNS:$nodename";
 268
 269     if ($rc && $rc->{search}) {
 270         $fqdn = $nodename . "." . $rc->{search};
 271         $names .= ",DNS:$fqdn";
 272     }
 273
 274     my $sslconf = <<__EOD;
 275 RANDFILE = /root/.rnd
 276 extensions = v3_req
 277
 278 [ req ]
 279 default_bits = 2048
 280 distinguished_name = req_distinguished_name
 281 req_extensions = v3_req
 282 prompt = no
 283 string_mask = nombstr
 284
 285 [ req_distinguished_name ]
 286 organizationalUnitName = PVE Cluster Node
 287 organizationName = Proxmox Virtual Environment
 288 commonName = $fqdn
 289
 290 [ v3_req ]
 291 basicConstraints = CA:FALSE
 292 extendedKeyUsage = serverAuth
 293 subjectAltName = $names
 294 __EOD
 295
 296     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 297     my $fh = IO::File->new ($cfgfn, "w");
 298     print $fh $sslconf;
 299     close ($fh);
 300
 301     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 302     unlink $reqfn;
 303
 304     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 305     eval {
 306         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 307                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 308     };
 309
 310     if (my $err = $@) {
 311         unlink $reqfn;
 312         unlink $cfgfn;
 313         die "unable to generate pve certificate request:\n$err";
 314     }
 315
 316     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 317
 318     eval {
 319         # wrap openssl with faketime to prevent bug #904
 320         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
 321                         '-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
 322                         '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
 323                         '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
 324     };
 325
 326     if (my $err = $@) {
 327         unlink $reqfn;
 328         unlink $cfgfn;
 329         die "unable to generate pve ssl certificate:\n$err";
 330     }
 331
 332     unlink $cfgfn;
 333     unlink $reqfn;
 334 }
 335
 336 sub gen_pve_node_files {
 337     my ($nodename, $ip, $opt_force) = @_;
 338
 339     gen_local_dirs($nodename);
 340
 341     gen_auth_key();
 342
 343     # make sure we have a (cluster wide) secret
 344     # for CSRFR prevention
 345     gen_pve_www_key();
 346
 347     # make sure we have a (per node) private key
 348     gen_pve_ssl_key($nodename);
 349
 350     # make sure we have a CA
 351     my $force = gen_pveca_cert();
 352
 353     $force = 1 if $opt_force;
 354
 355     gen_pve_ssl_cert($force, $nodename, $ip);
 356 }
 357
 358 my $vzdump_cron_dummy = <<__EOD;
 359 # cluster wide vzdump cron schedule
 360 # Atomatically generated file - do not edit
 361
 362 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 363
 364 __EOD
 365
 366 sub gen_pve_vzdump_symlink {
 367
 368     my $filename = "/etc/pve/vzdump.cron";
 369
 370     my $link_fn = "/etc/cron.d/vzdump";
 371
 372     if ((-f $filename) && (! -l $link_fn)) {
 373         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 374         symlink($filename, $link_fn);
 375     }
 376 }
 377
 378 sub gen_pve_vzdump_files {
 379
 380     my $filename = "/etc/pve/vzdump.cron";
 381
 382     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 383         if ! -f $filename;
 384
 385     gen_pve_vzdump_symlink();
 386 };
 387
 388 my $versions = {};
 389 my $vmlist = {};
 390 my $clinfo = {};
 391
 392 my $ipcc_send_rec = sub {
 393     my ($msgid, $data) = @_;
 394
 395     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 396
 397     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 398
 399     return $res;
 400 };
 401
 402 my $ipcc_send_rec_json = sub {
 403     my ($msgid, $data) = @_;
 404
 405     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 406
 407     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 408
 409     return decode_json($res);
 410 };
 411
 412 my $ipcc_get_config = sub {
 413     my ($path) = @_;
 414
 415     my $bindata = pack "Z*", $path;
 416     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_CONFIG, $bindata);
 417     if (!defined($res)) {
 418         if ($! != 0) {
 419             return undef if $! == ENOENT;
 420             die "$!\n";
 421         }
 422         return '';
 423     }
 424
 425     return $res;
 426 };
 427
 428 my $ipcc_get_status = sub {
 429     my ($name, $nodename) = @_;
 430
 431     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 432     return PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_STATUS, $bindata);
 433 };
 434
 435 my $ipcc_remove_status = sub {
 436     my ($name) = @_;
 437     # we just omit the data payload, pmxcfs takes this as hint and removes this
 438     # key from the status hashtable
 439     my $bindata = pack "Z[256]", $name;
 440     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 441 };
 442
 443 my $ipcc_update_status = sub {
 444     my ($name, $data) = @_;
 445
 446     my $raw = ref($data) ? encode_json($data) : $data;
 447     # update status
 448     my $bindata = pack "Z[256]Z*", $name, $raw;
 449
 450     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 451 };
 452
 453 my $ipcc_log = sub {
 454     my ($priority, $ident, $tag, $msg) = @_;
 455
 456     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 457     bytes::length($tag) + 1, $ident, $tag, $msg;
 458
 459     return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG, $bindata);
 460 };
 461
 462 my $ipcc_get_cluster_log = sub {
 463     my ($user, $max) = @_;
 464
 465     $max = 0 if !defined($max);
 466
 467     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 468     return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG, $bindata);
 469 };
 470
 471 my $ccache = {};
 472
 473 sub cfs_update {
 474     my ($fail) = @_;
 475     eval {
 476         my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION);
 477         die "no starttime\n" if !$res->{starttime};
 478
 479         if (!$res->{starttime} || !$versions->{starttime} ||
 480             $res->{starttime} != $versions->{starttime}) {
 481             #print "detected changed starttime\n";
 482             $vmlist = {};
 483             $clinfo = {};
 484             $ccache = {};
 485         }
 486
 487         $versions = $res;
 488     };
 489     my $err = $@;
 490     if ($err) {
 491         $versions = {};
 492         $vmlist = {};
 493         $clinfo = {};
 494         $ccache = {};
 495         die $err if $fail;
 496         warn $err;
 497     }
 498
 499     eval {
 500         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 501             #warn "detected new clinfo\n";
 502             $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO);
 503         }
 504     };
 505     $err = $@;
 506     if ($err) {
 507         $clinfo = {};
 508         die $err if $fail;
 509         warn $err;
 510     }
 511
 512     eval {
 513         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 514             #warn "detected new vmlist1\n";
 515             $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST);
 516         }
 517     };
 518     $err = $@;
 519     if ($err) {
 520         $vmlist = {};
 521         die $err if $fail;
 522         warn $err;
 523     }
 524 }
 525
 526 sub get_vmlist {
 527     return $vmlist;
 528 }
 529
 530 sub get_clinfo {
 531     return $clinfo;
 532 }
 533
 534 sub get_members {
 535     return $clinfo->{nodelist};
 536 }
 537
 538 sub get_nodelist {
 539     my $nodelist = $clinfo->{nodelist};
 540
 541     my $nodename = PVE::INotify::nodename();
 542
 543     if (!$nodelist || !$nodelist->{$nodename}) {
 544         return [ $nodename ];
 545     }
 546
 547     return [ keys %$nodelist ];
 548 }
 549
 550 # only stored in a in-memory hashtable inside pmxcfs, local data is gone after
 551 # a restart (of pmxcfs or the node), peer data is still available then
 552 # best used for status data, like running (ceph) services, package versions, ...
 553 sub broadcast_node_kv {
 554     my ($key, $data) = @_;
 555
 556     if (!defined($data)) {
 557         eval {
 558             $ipcc_remove_status->("kv/$key");
 559         };
 560     } else {
 561         die "cannot send a reference\n" if ref($data);
 562         my $size = length($data);
 563         die "data for '$key' too big\n" if $size >= (32 * 1024); # limit from pmxfs
 564
 565         eval {
 566             $ipcc_update_status->("kv/$key", $data);
 567         };
 568     }
 569
 570     warn $@ if $@;
 571 }
 572
 573 # nodename is optional
 574 sub get_node_kv {
 575     my ($key, $nodename) = @_;
 576
 577     my $res = {};
 578     my $get_node_data = sub {
 579         my ($node) = @_;
 580         my $raw = $ipcc_get_status->("kv/$key", $node);
 581         $res->{$node} = unpack("Z*", $raw) if $raw;
 582     };
 583
 584     if ($nodename) {
 585         $get_node_data->($nodename);
 586     } else {
 587         my $nodelist = get_nodelist();
 588
 589         foreach my $node (@$nodelist) {
 590             $get_node_data->($node);
 591         }
 592     }
 593
 594     return $res;
 595 }
 596
 597 # property: a config property you want to get, e.g., this is perfect to get
 598 # the 'lock' entry of a guest _fast_ (>100 faster than manual parsing here)
 599 # vmid: optipnal, if a valid is passed we only check that one, else return all
 600 # NOTE: does *not* searches snapshot and PENDING entries sections!
 601 sub get_guest_config_property {
 602     my ($property, $vmid) = @_;
 603
 604     die "property is required" if !defined($property);
 605
 606     my $bindata = pack "VZ*", $vmid // 0, $property;
 607     my $res = $ipcc_send_rec_json->(CFS_IPC_GET_GUEST_CONFIG_PROPERTY, $bindata);
 608
 609     return $res;
 610 }
 611
 612 # $data must be a chronological descending ordered array of tasks
 613 sub broadcast_tasklist {
 614     my ($data) = @_;
 615
 616     # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
 617     # from pmxcfs) - drop older items until we satisfy this constraint
 618     my $size = length(encode_json($data));
 619     while ($size >= (32 * 1024)) {
 620         pop @$data;
 621         $size = length(encode_json($data));
 622     }
 623
 624     eval {
 625         &$ipcc_update_status("tasklist", $data);
 626     };
 627
 628     warn $@ if $@;
 629 }
 630
 631 my $tasklistcache = {};
 632
 633 sub get_tasklist {
 634     my ($nodename) = @_;
 635
 636     my $kvstore = $versions->{kvstore} || {};
 637
 638     my $nodelist = get_nodelist();
 639
 640     my $res = [];
 641     foreach my $node (@$nodelist) {
 642         next if $nodename && ($nodename ne $node);
 643         eval {
 644             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 645             my $cd = $tasklistcache->{$node};
 646             if (!$cd || !$ver || !$cd->{version} ||
 647                 ($cd->{version} != $ver)) {
 648                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 649                 my $data = decode_json($raw);
 650                 push @$res, @$data;
 651                 $cd = $tasklistcache->{$node} = {
 652                     data => $data,
 653                     version => $ver,
 654                 };
 655             } elsif ($cd && $cd->{data}) {
 656                 push @$res, @{$cd->{data}};
 657             }
 658         };
 659         my $err = $@;
 660         syslog('err', $err) if $err;
 661     }
 662
 663     return $res;
 664 }
 665
 666 sub broadcast_rrd {
 667     my ($rrdid, $data) = @_;
 668
 669     eval {
 670         &$ipcc_update_status("rrd/$rrdid", $data);
 671     };
 672     my $err = $@;
 673
 674     warn $err if $err;
 675 }
 676
 677 my $last_rrd_dump = 0;
 678 my $last_rrd_data = "";
 679
 680 sub rrd_dump {
 681
 682     my $ctime = time();
 683
 684     my $diff = $ctime - $last_rrd_dump;
 685     if ($diff < 2) {
 686         return $last_rrd_data;
 687     }
 688
 689     my $raw;
 690     eval {
 691         $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP);
 692     };
 693     my $err = $@;
 694
 695     if ($err) {
 696         warn $err;
 697         return {};
 698     }
 699
 700     my $res = {};
 701
 702     if ($raw) {
 703         while ($raw =~ s/^(.*)\n//) {
 704             my ($key, @ela) = split(/:/, $1);
 705             next if !$key;
 706             next if !(scalar(@ela) > 1);
 707             $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
 708         }
 709     }
 710
 711     $last_rrd_dump = $ctime;
 712     $last_rrd_data = $res;
 713
 714     return $res;
 715 }
 716
 717 sub create_rrd_data {
 718     my ($rrdname, $timeframe, $cf) = @_;
 719
 720     my $rrddir = "/var/lib/rrdcached/db";
 721
 722     my $rrd = "$rrddir/$rrdname";
 723
 724     my $setup = {
 725         hour =>  [ 60, 70 ],
 726         day  =>  [ 60*30, 70 ],
 727         week =>  [ 60*180, 70 ],
 728         month => [ 60*720, 70 ],
 729         year =>  [ 60*10080, 70 ],
 730     };
 731
 732     my ($reso, $count) = @{$setup->{$timeframe}};
 733     my $ctime  = $reso*int(time()/$reso);
 734     my $req_start = $ctime - $reso*$count;
 735
 736     $cf = "AVERAGE" if !$cf;
 737
 738     my @args = (
 739         "-s" => $req_start,
 740         "-e" => $ctime - 1,
 741         "-r" => $reso,
 742         );
 743
 744     my $socket = "/var/run/rrdcached.sock";
 745     push @args, "--daemon" => "unix:$socket" if -S $socket;
 746
 747     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 748
 749     my $err = RRDs::error;
 750     die "RRD error: $err\n" if $err;
 751
 752     die "got wrong time resolution ($step != $reso)\n"
 753         if $step != $reso;
 754
 755     my $res = [];
 756     my $fields = scalar(@$names);
 757     for my $line (@$data) {
 758         my $entry = { 'time' => $start };
 759         $start += $step;
 760         for (my $i = 0; $i < $fields; $i++) {
 761             my $name = $names->[$i];
 762             if (defined(my $val = $line->[$i])) {
 763                 $entry->{$name} = $val;
 764             } else {
 765                 # leave empty fields undefined
 766                 # maybe make this configurable?
 767             }
 768         }
 769         push @$res, $entry;
 770     }
 771
 772     return $res;
 773 }
 774
 775 sub create_rrd_graph {
 776     my ($rrdname, $timeframe, $ds, $cf) = @_;
 777
 778     # Using RRD graph is clumsy - maybe it
 779     # is better to simply fetch the data, and do all display
 780     # related things with javascript (new extjs html5 graph library).
 781
 782     my $rrddir = "/var/lib/rrdcached/db";
 783
 784     my $rrd = "$rrddir/$rrdname";
 785
 786     my @ids = PVE::Tools::split_list($ds);
 787
 788     my $ds_txt = join('_', @ids);
 789
 790     my $filename = "${rrd}_${ds_txt}.png";
 791
 792     my $setup = {
 793         hour =>  [ 60, 60 ],
 794         day  =>  [ 60*30, 70 ],
 795         week =>  [ 60*180, 70 ],
 796         month => [ 60*720, 70 ],
 797         year =>  [ 60*10080, 70 ],
 798     };
 799
 800     my ($reso, $count) = @{$setup->{$timeframe}};
 801
 802     my @args = (
 803         "--imgformat" => "PNG",
 804         "--border" => 0,
 805         "--height" => 200,
 806         "--width" => 800,
 807         "--start" => - $reso*$count,
 808         "--end" => 'now' ,
 809         "--lower-limit" => 0,
 810         );
 811
 812     my $socket = "/var/run/rrdcached.sock";
 813     push @args, "--daemon" => "unix:$socket" if -S $socket;
 814
 815     my @coldef = ('#00ddff', '#ff0000');
 816
 817     $cf = "AVERAGE" if !$cf;
 818
 819     my $i = 0;
 820     foreach my $id (@ids) {
 821         my $col = $coldef[$i++] || die "fixme: no color definition";
 822         push @args, "DEF:${id}=$rrd:${id}:$cf";
 823         my $dataid = $id;
 824         if ($id eq 'cpu' || $id eq 'iowait') {
 825             push @args, "CDEF:${id}_per=${id},100,*";
 826             $dataid = "${id}_per";
 827         }
 828         push @args, "LINE2:${dataid}${col}:${id}";
 829     }
 830
 831     push @args, '--full-size-mode';
 832
 833     # we do not really store data into the file
 834     my $res = RRDs::graphv('-', @args);
 835
 836     my $err = RRDs::error;
 837     die "RRD error: $err\n" if $err;
 838
 839     return { filename => $filename, image => $res->{image} };
 840 }
 841
 842 # a fast way to read files (avoid fuse overhead)
 843 sub get_config {
 844     my ($path) = @_;
 845
 846     return &$ipcc_get_config($path);
 847 }
 848
 849 sub get_cluster_log {
 850     my ($user, $max) = @_;
 851
 852     return &$ipcc_get_cluster_log($user, $max);
 853 }
 854
 855 my $file_info = {};
 856
 857 sub cfs_register_file {
 858     my ($filename, $parser, $writer) = @_;
 859
 860     $observed->{$filename} || die "unknown file '$filename'";
 861
 862     die "file '$filename' already registered" if $file_info->{$filename};
 863
 864     $file_info->{$filename} = {
 865         parser => $parser,
 866         writer => $writer,
 867     };
 868 }
 869
 870 my $ccache_read = sub {
 871     my ($filename, $parser, $version) = @_;
 872
 873     $ccache->{$filename} = {} if !$ccache->{$filename};
 874
 875     my $ci = $ccache->{$filename};
 876
 877     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 878         # we always call the parser, even when the file does not exists
 879         # (in that case $data is undef)
 880         my $data = get_config($filename);
 881         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 882         $ci->{version} = $version;
 883     }
 884
 885     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 886
 887     return $res;
 888 };
 889
 890 sub cfs_file_version {
 891     my ($filename) = @_;
 892
 893     my $version;
 894     my $infotag;
 895     if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
 896         my ($type, $vmid) = ($1, $2);
 897         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 898             $version = $vmlist->{ids}->{$vmid}->{version};
 899         }
 900         $infotag = "/$type/";
 901     } else {
 902         $infotag = $filename;
 903         $version = $versions->{$filename};
 904     }
 905
 906     my $info = $file_info->{$infotag} ||
 907         die "unknown file type '$filename'\n";
 908
 909     return wantarray ? ($version, $info) : $version;
 910 }
 911
 912 sub cfs_read_file {
 913     my ($filename) = @_;
 914
 915     my ($version, $info) = cfs_file_version($filename);
 916     my $parser = $info->{parser};
 917
 918     return &$ccache_read($filename, $parser, $version);
 919 }
 920
 921 sub cfs_write_file {
 922     my ($filename, $data) = @_;
 923
 924     my ($version, $info) = cfs_file_version($filename);
 925
 926     my $writer = $info->{writer} || die "no writer defined";
 927
 928     my $fsname = "/etc/pve/$filename";
 929
 930     my $raw = &$writer($fsname, $data);
 931
 932     if (my $ci = $ccache->{$filename}) {
 933         $ci->{version} = undef;
 934     }
 935
 936     PVE::Tools::file_set_contents($fsname, $raw);
 937 }
 938
 939 my $cfs_lock = sub {
 940     my ($lockid, $timeout, $code, @param) = @_;
 941
 942     my $prev_alarm = alarm(0); # suspend outer alarm early
 943
 944     my $res;
 945     my $got_lock = 0;
 946
 947     # this timeout is for acquire the lock
 948     $timeout = 10 if !$timeout;
 949
 950     my $filename = "$lockdir/$lockid";
 951
 952     eval {
 953
 954         mkdir $lockdir;
 955
 956         if (! -d $lockdir) {
 957             die "pve cluster filesystem not online.\n";
 958         }
 959
 960         my $timeout_err = sub { die "got lock request timeout\n"; };
 961         local $SIG{ALRM} = $timeout_err;
 962
 963         while (1) {
 964             alarm ($timeout);
 965             $got_lock = mkdir($filename);
 966             $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
 967
 968             last if $got_lock;
 969
 970             $timeout_err->() if $timeout <= 0;
 971
 972             print STDERR "trying to acquire cfs lock '$lockid' ...\n";
 973             utime (0, 0, $filename); # cfs unlock request
 974             sleep(1);
 975         }
 976
 977         # fixed command timeout: cfs locks have a timeout of 120
 978         # using 60 gives us another 60 seconds to abort the task
 979         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 980         alarm(60);
 981
 982         cfs_update(); # make sure we read latest versions inside code()
 983
 984         $res = &$code(@param);
 985
 986         alarm(0);
 987     };
 988
 989     my $err = $@;
 990
 991     $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
 992
 993     rmdir $filename if $got_lock; # if we held the lock always unlock again
 994
 995     alarm($prev_alarm);
 996
 997     if ($err) {
 998         $@ = "error with cfs lock '$lockid': $err";
 999         return undef;
1000     }
1001
1002     $@ = undef;
1003
1004     return $res;
1005 };
1006
1007 sub cfs_lock_file {
1008     my ($filename, $timeout, $code, @param) = @_;
1009
1010     my $info = $observed->{$filename} || die "unknown file '$filename'";
1011
1012     my $lockid = "file-$filename";
1013     $lockid =~ s/[.\/]/_/g;
1014
1015     &$cfs_lock($lockid, $timeout, $code, @param);
1016 }
1017
1018 sub cfs_lock_storage {
1019     my ($storeid, $timeout, $code, @param) = @_;
1020
1021     my $lockid = "storage-$storeid";
1022
1023     &$cfs_lock($lockid, $timeout, $code, @param);
1024 }
1025
1026 sub cfs_lock_domain {
1027     my ($domainname, $timeout, $code, @param) = @_;
1028
1029     my $lockid = "domain-$domainname";
1030
1031     &$cfs_lock($lockid, $timeout, $code, @param);
1032 }
1033
1034 sub cfs_lock_acme {
1035     my ($account, $timeout, $code, @param) = @_;
1036
1037     my $lockid = "acme-$account";
1038
1039     &$cfs_lock($lockid, $timeout, $code, @param);
1040 }
1041
1042 sub cfs_lock_authkey {
1043     my ($timeout, $code, @param) = @_;
1044
1045     $cfs_lock->('authkey', $timeout, $code, @param);
1046 }
1047
1048 my $log_levels = {
1049     "emerg" => 0,
1050     "alert" => 1,
1051     "crit" => 2,
1052     "critical" => 2,
1053     "err" => 3,
1054     "error" => 3,
1055     "warn" => 4,
1056     "warning" => 4,
1057     "notice" => 5,
1058     "info" => 6,
1059     "debug" => 7,
1060 };
1061
1062 sub log_msg {
1063    my ($priority, $ident, $msg) = @_;
1064
1065    if (my $tmp = $log_levels->{$priority}) {
1066        $priority = $tmp;
1067    }
1068
1069    die "need numeric log priority" if $priority !~ /^\d+$/;
1070
1071    my $tag = PVE::SafeSyslog::tag();
1072
1073    $msg = "empty message" if !$msg;
1074
1075    $ident = "" if !$ident;
1076    $ident = encode("ascii", $ident,
1077                    sub { sprintf "\\u%04x", shift });
1078
1079    my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
1080
1081    if ($ident) {
1082        syslog($priority, "<%s> %s", $ident, $ascii);
1083    } else {
1084        syslog($priority, "%s", $ascii);
1085    }
1086
1087    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
1088
1089    syslog("err", "writing cluster log failed: $@") if $@;
1090 }
1091
1092 sub check_vmid_unused {
1093     my ($vmid, $noerr) = @_;
1094
1095     my $vmlist = get_vmlist();
1096
1097     my $d = $vmlist->{ids}->{$vmid};
1098     return 1 if !defined($d);
1099
1100     return undef if $noerr;
1101
1102     my $vmtypestr =  $d->{type} eq 'qemu' ? 'VM' : 'CT';
1103     die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
1104 }
1105
1106 sub check_node_exists {
1107     my ($nodename, $noerr) = @_;
1108
1109     my $nodelist = $clinfo->{nodelist};
1110     return 1 if $nodelist && $nodelist->{$nodename};
1111
1112     return undef if $noerr;
1113
1114     die "no such cluster node '$nodename'\n";
1115 }
1116
1117 # this is also used to get the IP of the local node
1118 sub remote_node_ip {
1119     my ($nodename, $noerr) = @_;
1120
1121     my $nodelist = $clinfo->{nodelist};
1122     if ($nodelist && $nodelist->{$nodename}) {
1123         if (my $ip = $nodelist->{$nodename}->{ip}) {
1124             return $ip if !wantarray;
1125             my $family = $nodelist->{$nodename}->{address_family};
1126             if (!$family) {
1127                 $nodelist->{$nodename}->{address_family} =
1128                     $family =
1129                     PVE::Tools::get_host_address_family($ip);
1130             }
1131             return wantarray ? ($ip, $family) : $ip;
1132         }
1133     }
1134
1135     # fallback: try to get IP by other means
1136     return PVE::Network::get_ip_from_hostname($nodename, $noerr);
1137 }
1138
1139 sub get_local_migration_ip {
1140     my ($migration_network, $noerr) = @_;
1141
1142     my $cidr = $migration_network;
1143
1144     if (!defined($cidr)) {
1145         my $dc_conf = cfs_read_file('datacenter.cfg');
1146         $cidr = $dc_conf->{migration}->{network}
1147           if defined($dc_conf->{migration}->{network});
1148     }
1149
1150     if (defined($cidr)) {
1151         my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
1152
1153         die "could not get migration ip: no IP address configured on local " .
1154             "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
1155
1156         die "could not get migration ip: multiple IP address configured for " .
1157             "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
1158
1159         return @$ips[0];
1160     }
1161
1162     return undef;
1163 };
1164
1165 # ssh related utility functions
1166
1167 sub ssh_merge_keys {
1168     # remove duplicate keys in $sshauthkeys
1169     # ssh-copy-id simply add keys, so the file can grow to large
1170
1171     my $data = '';
1172     if (-f $sshauthkeys) {
1173         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
1174         chomp($data);
1175     }
1176
1177     my $found_backup;
1178     if (-f $rootsshauthkeysbackup) {
1179         $data .= "\n";
1180         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
1181         chomp($data);
1182         $found_backup = 1;
1183     }
1184
1185     # always add ourself
1186     if (-f $ssh_rsa_id) {
1187         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
1188         chomp($pub);
1189         $data .= "\n$pub\n";
1190     }
1191
1192     my $newdata = "";
1193     my $vhash = {};
1194     my @lines = split(/\n/, $data);
1195     foreach my $line (@lines) {
1196         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1197             next if $vhash->{$3}++;
1198         }
1199         $newdata .= "$line\n";
1200     }
1201
1202     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
1203
1204     if ($found_backup && -l $rootsshauthkeys) {
1205         # everything went well, so we can remove the backup
1206         unlink $rootsshauthkeysbackup;
1207     }
1208 }
1209
1210 sub setup_sshd_config {
1211     my () = @_;
1212
1213     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
1214
1215     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1216
1217     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1218         chomp $conf;
1219         $conf .= "\nPermitRootLogin yes\n";
1220     }
1221
1222     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
1223
1224     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
1225 }
1226
1227 sub setup_rootsshconfig {
1228
1229     # create ssh key if it does not exist
1230     if (! -f $ssh_rsa_id) {
1231         mkdir '/root/.ssh/';
1232         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1233     }
1234
1235     # create ssh config if it does not exist
1236     if (! -f $rootsshconfig) {
1237         mkdir '/root/.ssh';
1238         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
1239             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
1240             # changed order to put AES before Chacha20 (most hardware has AESNI)
1241             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
1242             close($fh);
1243         }
1244     }
1245 }
1246
1247 sub setup_ssh_keys {
1248
1249     mkdir $authdir;
1250
1251     my $import_ok;
1252
1253     if (! -f $sshauthkeys) {
1254         my $old;
1255         if (-f $rootsshauthkeys) {
1256             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
1257         }
1258         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1259             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
1260             close($fh);
1261             $import_ok = 1;
1262         }
1263     }
1264
1265     warn "can't create shared ssh key database '$sshauthkeys'\n"
1266         if ! -f $sshauthkeys;
1267
1268     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
1269         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1270             warn "rename $rootsshauthkeys failed - $!\n";
1271         }
1272     }
1273
1274     if (! -l $rootsshauthkeys) {
1275         symlink $sshauthkeys, $rootsshauthkeys;
1276     }
1277
1278     if (! -l $rootsshauthkeys) {
1279         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1280     } else {
1281         unlink $rootsshauthkeysbackup if $import_ok;
1282     }
1283 }
1284
1285 sub ssh_unmerge_known_hosts {
1286     return if ! -l $sshglobalknownhosts;
1287
1288     my $old = '';
1289     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1290         if -f $sshknownhosts;
1291
1292     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1293 }
1294
1295 sub ssh_merge_known_hosts {
1296     my ($nodename, $ip_address, $createLink) = @_;
1297
1298     die "no node name specified" if !$nodename;
1299     die "no ip address specified" if !$ip_address;
1300
1301     # ssh lowercases hostnames (aliases) before comparision, so we need too
1302     $nodename = lc($nodename);
1303     $ip_address = lc($ip_address);
1304
1305     mkdir $authdir;
1306
1307     if (! -f $sshknownhosts) {
1308         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1309             close($fh);
1310         }
1311     }
1312
1313     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1314
1315     my $new = '';
1316
1317     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1318         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1319     }
1320
1321     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1322     # Note: file sometimes containe emty lines at start, so we use multiline match
1323     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
1324     $hostkey = $1;
1325
1326     my $data = '';
1327     my $vhash = {};
1328
1329     my $found_nodename;
1330     my $found_local_ip;
1331
1332     my $merge_line = sub {
1333         my ($line, $all) = @_;
1334
1335         return if $line =~ m/^\s*$/; # skip empty lines
1336         return if $line =~ m/^#/; # skip comments
1337
1338         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1339             my $key = $1;
1340             my $rsakey = $2;
1341             if (!$vhash->{$key}) {
1342                 $vhash->{$key} = 1;
1343                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1344                     my $salt = decode_base64($1);
1345                     my $digest = $2;
1346                     my $hmac = Digest::HMAC_SHA1->new($salt);
1347                     $hmac->add($nodename);
1348                     my $hd = $hmac->b64digest . '=';
1349                     if ($digest eq $hd) {
1350                         if ($rsakey eq $hostkey) {
1351                             $found_nodename = 1;
1352                             $data .= $line;
1353                         }
1354                         return;
1355                     }
1356                     $hmac = Digest::HMAC_SHA1->new($salt);
1357                     $hmac->add($ip_address);
1358                     $hd = $hmac->b64digest . '=';
1359                     if ($digest eq $hd) {
1360                         if ($rsakey eq $hostkey) {
1361                             $found_local_ip = 1;
1362                             $data .= $line;
1363                         }
1364                         return;
1365                     }
1366                 } else {
1367                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
1368                     if ($key eq $ip_address) {
1369                         $found_local_ip = 1 if $rsakey eq $hostkey;
1370                     } elsif ($key eq $nodename) {
1371                         $found_nodename = 1 if $rsakey eq $hostkey;
1372                     }
1373                 }
1374                 $data .= $line;
1375             }
1376         } elsif ($all) {
1377             $data .= $line;
1378         }
1379     };
1380
1381     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1382         my $line = "$2\n";
1383         &$merge_line($line, 1);
1384     }
1385
1386     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1387         my $line = "$2\n";
1388         &$merge_line($line);
1389     }
1390
1391     # add our own key if not already there
1392     $data .= "$nodename $hostkey\n" if !$found_nodename;
1393     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
1394
1395     PVE::Tools::file_set_contents($sshknownhosts, $data);
1396
1397     return if !$createLink;
1398
1399     unlink $sshglobalknownhosts;
1400     symlink $sshknownhosts, $sshglobalknownhosts;
1401
1402     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1403         if ! -l $sshglobalknownhosts;
1404
1405 }
1406
1407 my $migration_format = {
1408     type => {
1409         default_key => 1,
1410         type => 'string',
1411         enum => ['secure', 'insecure'],
1412         description => "Migration traffic is encrypted using an SSH tunnel by " .
1413           "default. On secure, completely private networks this can be " .
1414           "disabled to increase performance.",
1415         default => 'secure',
1416     },
1417     network => {
1418         optional => 1,
1419         type => 'string', format => 'CIDR',
1420         format_description => 'CIDR',
1421         description => "CIDR of the (sub) network that is used for migration."
1422     },
1423 };
1424
1425 my $ha_format = {
1426     shutdown_policy => {
1427         type => 'string',
1428         enum => ['freeze', 'failover', 'conditional'],
1429         description => "The policy for HA services on node shutdown. 'freeze' disables auto-recovery, 'failover' ensures recovery, 'conditional' recovers on poweroff and freezes on reboot. Running HA Services will always get stopped first on shutdown.",
1430         verbose_description => "Describes the policy for handling HA services on poweroff or reboot of a node. Freeze will always freeze services which are still located on the node on shutdown, those services won't be recovered by the HA manager. Failover will not mark the services as frozen and thus the services will get recovered to other nodes, if the shutdown node does not come up again quickly (< 1min). 'conditional' chooses automatically depending on the type of shutdown, i.e., on a reboot the service will be frozen but on a poweroff the service will stay as is, and thus get recovered after about 2 minutes.",
1431         default => 'conditional',
1432     }
1433 };
1434
1435 PVE::JSONSchema::register_format('mac-prefix', \&pve_verify_mac_prefix);
1436 sub pve_verify_mac_prefix {
1437     my ($mac_prefix, $noerr) = @_;
1438
1439     if ($mac_prefix !~ m/^[a-f0-9][02468ace](?::[a-f0-9]{2}){0,2}:?$/i) {
1440         return undef if $noerr;
1441         die "value is not a valid unicast MAC address prefix\n";
1442     }
1443     return $mac_prefix;
1444 }
1445
1446 our $u2f_format = {
1447     appid => {
1448         type => 'string',
1449         description => "U2F AppId URL override. Defaults to the origin.",
1450         format_description => 'APPID',
1451         optional => 1,
1452     },
1453     origin => {
1454         type => 'string',
1455         description => "U2F Origin override. Mostly useful for single nodes with a single URL.",
1456         format_description => 'URL',
1457         optional => 1,
1458     },
1459 };
1460
1461 my $datacenter_schema = {
1462     type => "object",
1463     additionalProperties => 0,
1464     properties => {
1465         keyboard => {
1466             optional => 1,
1467             type => 'string',
1468             description => "Default keybord layout for vnc server.",
1469             enum => PVE::Tools::kvmkeymaplist(),
1470         },
1471         language => {
1472             optional => 1,
1473             type => 'string',
1474             description => "Default GUI language.",
1475             enum => [
1476                 'ca',
1477                 'da',
1478                 'de',
1479                 'en',
1480                 'es',
1481                 'eu',
1482                 'fa',
1483                 'fr',
1484                 'he',
1485                 'it',
1486                 'ja',
1487                 'nb',
1488                 'nn',
1489                 'pl',
1490                 'pt_BR',
1491                 'ru',
1492                 'sl',
1493                 'sv',
1494                 'tr',
1495                 'zh_CN',
1496                 'zh_TW',
1497             ],
1498         },
1499         http_proxy => {
1500             optional => 1,
1501             type => 'string',
1502             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1503             pattern => "http://.*",
1504         },
1505         migration_unsecure => {
1506             optional => 1,
1507             type => 'boolean',
1508             description => "Migration is secure using SSH tunnel by default. " .
1509               "For secure private networks you can disable it to speed up " .
1510               "migration. Deprecated, use the 'migration' property instead!",
1511         },
1512         migration => {
1513             optional => 1,
1514             type => 'string', format => $migration_format,
1515             description => "For cluster wide migration settings.",
1516         },
1517         console => {
1518             optional => 1,
1519             type => 'string',
1520             description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), an HTML5 based vnc viewer (noVNC), or an HTML5 based console client (xtermjs). If the selected viewer is not available (e.g. SPICE not activated for the VM), the fallback is noVNC.",
1521             enum => ['applet', 'vv', 'html5', 'xtermjs'],
1522         },
1523         email_from => {
1524             optional => 1,
1525             type => 'string',
1526             format => 'email-opt',
1527             description => "Specify email address to send notification from (default is root@\$hostname)",
1528         },
1529         max_workers => {
1530             optional => 1,
1531             type => 'integer',
1532             minimum => 1,
1533             description => "Defines how many workers (per node) are maximal started ".
1534               " on actions like 'stopall VMs' or task from the ha-manager.",
1535         },
1536         fencing => {
1537             optional => 1,
1538             type => 'string',
1539             default => 'watchdog',
1540             enum => [ 'watchdog', 'hardware', 'both' ],
1541             description => "Set the fencing mode of the HA cluster. Hardware mode " .
1542               "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
1543               " With both all two modes are used." .
1544               "\n\nWARNING: 'hardware' and 'both' are EXPERIMENTAL & WIP",
1545         },
1546         ha => {
1547             optional => 1,
1548             type => 'string', format => $ha_format,
1549             description => "Cluster wide HA settings.",
1550         },
1551         mac_prefix => {
1552             optional => 1,
1553             type => 'string',
1554             format => 'mac-prefix',
1555             description => 'Prefix for autogenerated MAC addresses.',
1556         },
1557         bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
1558         u2f => {
1559             optional => 1,
1560             type => 'string',
1561             format => $u2f_format,
1562             description => 'u2f',
1563         },
1564     },
1565 };
1566
1567 # make schema accessible from outside (for documentation)
1568 sub get_datacenter_schema { return $datacenter_schema };
1569
1570 sub parse_datacenter_config {
1571     my ($filename, $raw) = @_;
1572
1573     my $res = PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
1574
1575     if (my $migration = $res->{migration}) {
1576         $res->{migration} = PVE::JSONSchema::parse_property_string($migration_format, $migration);
1577     }
1578
1579     if (my $ha = $res->{ha}) {
1580         $res->{ha} = PVE::JSONSchema::parse_property_string($ha_format, $ha);
1581     }
1582
1583     # for backwards compatibility only, new migration property has precedence
1584     if (defined($res->{migration_unsecure})) {
1585         if (defined($res->{migration}->{type})) {
1586             warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
1587               "set at same time! Ignore 'migration_unsecure'\n";
1588         } else {
1589             $res->{migration}->{type} = ($res->{migration_unsecure}) ? 'insecure' : 'secure';
1590         }
1591     }
1592
1593     # for backwards compatibility only, applet maps to html5
1594     if (defined($res->{console}) && $res->{console} eq 'applet') {
1595         $res->{console} = 'html5';
1596     }
1597
1598     return $res;
1599 }
1600
1601 sub write_datacenter_config {
1602     my ($filename, $cfg) = @_;
1603
1604     # map deprecated setting to new one
1605     if (defined($cfg->{migration_unsecure}) && !defined($cfg->{migration})) {
1606         my $migration_unsecure = delete $cfg->{migration_unsecure};
1607         $cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
1608     }
1609
1610     # map deprecated applet setting to html5
1611     if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
1612         $cfg->{console} = 'html5';
1613     }
1614
1615     if (ref($cfg->{migration})) {
1616         my $migration = $cfg->{migration};
1617         $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
1618     }
1619
1620     if (ref($cfg->{ha})) {
1621         my $ha = $cfg->{ha};
1622         $cfg->{ha} = PVE::JSONSchema::print_property_string($ha, $ha_format);
1623     }
1624
1625     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1626 }
1627
1628 cfs_register_file('datacenter.cfg',
1629                   \&parse_datacenter_config,
1630                   \&write_datacenter_config);
1631
1632 # X509 Certificate cache helper
1633
1634 my $cert_cache_nodes = {};
1635 my $cert_cache_timestamp = time();
1636 my $cert_cache_fingerprints = {};
1637
1638 sub update_cert_cache {
1639     my ($update_node, $clear) = @_;
1640
1641     syslog('info', "Clearing outdated entries from certificate cache")
1642         if $clear;
1643
1644     $cert_cache_timestamp = time() if !defined($update_node);
1645
1646     my $node_list = defined($update_node) ?
1647         [ $update_node ] : [ keys %$cert_cache_nodes ];
1648
1649     foreach my $node (@$node_list) {
1650         my $clear_old = sub {
1651             if (my $old_fp = $cert_cache_nodes->{$node}) {
1652                 # distrust old fingerprint
1653                 delete $cert_cache_fingerprints->{$old_fp};
1654                 # ensure reload on next proxied request
1655                 delete $cert_cache_nodes->{$node};
1656             }
1657         };
1658
1659         my $fp = eval { get_node_fingerprint($node) };
1660         if (my $err = $@) {
1661             warn "$err\n";
1662             &$clear_old() if $clear;
1663             next;
1664         }
1665
1666         my $old_fp = $cert_cache_nodes->{$node};
1667         $cert_cache_fingerprints->{$fp} = 1;
1668         $cert_cache_nodes->{$node} = $fp;
1669
1670         if (defined($old_fp) && $fp ne $old_fp) {
1671             delete $cert_cache_fingerprints->{$old_fp};
1672         }
1673     }
1674 }
1675
1676 # load and cache cert fingerprint once
1677 sub initialize_cert_cache {
1678     my ($node) = @_;
1679
1680     update_cert_cache($node)
1681         if defined($node) && !defined($cert_cache_nodes->{$node});
1682 }
1683
1684 sub read_ssl_cert_fingerprint {
1685     my ($cert_path) = @_;
1686
1687     my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
1688         or die "unable to read '$cert_path' - $!\n";
1689
1690     my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
1691     Net::SSLeay::BIO_free($bio);
1692
1693     die "unable to read certificate from '$cert_path'\n" if !$cert;
1694
1695     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1696     Net::SSLeay::X509_free($cert);
1697
1698     die "unable to get fingerprint for '$cert_path' - got empty value\n"
1699         if !defined($fp) || $fp eq '';
1700
1701     return $fp;
1702 }
1703
1704 sub get_node_fingerprint {
1705     my ($node) = @_;
1706
1707     my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
1708     my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
1709
1710     $cert_path = $custom_cert_path if -f $custom_cert_path;
1711
1712     return read_ssl_cert_fingerprint($cert_path);
1713 }
1714
1715
1716 sub check_cert_fingerprint {
1717     my ($cert) = @_;
1718
1719     # clear cache every 30 minutes at least
1720     update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
1721
1722     # get fingerprint of server certificate
1723     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1724     return 0 if !defined($fp) || $fp eq ''; # error
1725
1726     my $check = sub {
1727         for my $expected (keys %$cert_cache_fingerprints) {
1728             return 1 if $fp eq $expected;
1729         }
1730         return 0;
1731     };
1732
1733     return 1 if &$check();
1734
1735     # clear cache and retry at most once every minute
1736     if (time() - $cert_cache_timestamp >= 60) {
1737         syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
1738         update_cert_cache();
1739         return &$check();
1740     }
1741
1742     return 0;
1743 }
1744
1745 # bash completion helpers
1746
1747 sub complete_next_vmid {
1748
1749     my $vmlist = get_vmlist() || {};
1750     my $idlist = $vmlist->{ids} || {};
1751
1752     for (my $i = 100; $i < 10000; $i++) {
1753         return [$i] if !defined($idlist->{$i});
1754     }
1755
1756     return [];
1757 }
1758
1759 sub complete_vmid {
1760
1761     my $vmlist = get_vmlist();
1762     my $ids = $vmlist->{ids} || {};
1763
1764     return [ keys %$ids ];
1765 }
1766
1767 sub complete_local_vmid {
1768
1769     my $vmlist = get_vmlist();
1770     my $ids = $vmlist->{ids} || {};
1771
1772     my $nodename = PVE::INotify::nodename();
1773
1774     my $res = [];
1775     foreach my $vmid (keys %$ids) {
1776         my $d = $ids->{$vmid};
1777         next if !$d->{node} || $d->{node} ne $nodename;
1778         push @$res, $vmid;
1779     }
1780
1781     return $res;
1782 }
1783
1784 sub complete_migration_target {
1785
1786     my $res = [];
1787
1788     my $nodename = PVE::INotify::nodename();
1789
1790     my $nodelist = get_nodelist();
1791     foreach my $node (@$nodelist) {
1792         next if $node eq $nodename;
1793         push @$res, $node;
1794     }
1795
1796     return $res;
1797 }
1798
1799 sub get_ssh_info {
1800     my ($node, $network_cidr) = @_;
1801
1802     my $ip;
1803     if (defined($network_cidr)) {
1804         # Use mtunnel via to get the remote node's ip inside $network_cidr.
1805         # This goes over the regular network (iow. uses get_ssh_info() with
1806         # $network_cidr undefined.
1807         # FIXME: Use the REST API client for this after creating an API entry
1808         # for get_migration_ip.
1809         my $default_remote = get_ssh_info($node, undef);
1810         my $default_ssh = ssh_info_to_command($default_remote);
1811         my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
1812             '-migration_network', $network_cidr,
1813             '-get_migration_ip'
1814         ];
1815         PVE::Tools::run_command($cmd, outfunc => sub {
1816             my ($line) = @_;
1817             chomp $line;
1818             die "internal error: unexpected output from mtunnel\n"
1819                 if defined($ip);
1820             if ($line =~ /^ip: '(.*)'$/) {
1821                 $ip = $1;
1822             } else {
1823                 die "internal error: bad output from mtunnel\n"
1824                     if defined($ip);
1825             }
1826         });
1827         die "failed to get ip for node '$node' in network '$network_cidr'\n"
1828             if !defined($ip);
1829     } else {
1830         $ip = remote_node_ip($node);
1831     }
1832
1833     return {
1834         ip => $ip,
1835         name => $node,
1836         network => $network_cidr,
1837     };
1838 }
1839
1840 sub ssh_info_to_command_base {
1841     my ($info, @extra_options) = @_;
1842     return [
1843         '/usr/bin/ssh',
1844         '-e', 'none',
1845         '-o', 'BatchMode=yes',
1846         '-o', 'HostKeyAlias='.$info->{name},
1847         @extra_options
1848     ];
1849 }
1850
1851 sub ssh_info_to_command {
1852     my ($info, @extra_options) = @_;
1853     my $cmd = ssh_info_to_command_base($info, @extra_options);
1854     push @$cmd, "root\@$info->{ip}";
1855     return $cmd;
1856 }
1857
1858 my $corosync_link_format = {
1859     address => {
1860         default_key => 1,
1861         type => 'string', format => 'address',
1862         format_description => 'IP',
1863         description => "Hostname (or IP) of this corosync link address.",
1864     },
1865     priority => {
1866         optional => 1,
1867         type => 'integer',
1868         minimum => 0,
1869         maximum => 255,
1870         default => 0,
1871         description => "The priority for the link when knet is used in 'passive' mode. Lower value means higher priority.",
1872     },
1873 };
1874 my $corosync_link_desc = {
1875     type => 'string', format => $corosync_link_format,
1876     description => "Address and priority information of a single corosync link.",
1877     optional => 1,
1878 };
1879 PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);
1880
1881 sub parse_corosync_link {
1882     my ($value) = @_;
1883
1884     return undef if !defined($value);
1885
1886     return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
1887 }
1888
1889 sub assert_joinable {
1890     my ($local_addr, $link0, $link1, $force) = @_;
1891
1892     my $errors = '';
1893     my $error = sub { $errors .= "* $_[0]\n"; };
1894
1895     if (-f $authfile) {
1896         $error->("authentication key '$authfile' already exists");
1897     }
1898
1899     if (-f $clusterconf)  {
1900         $error->("cluster config '$clusterconf' already exists");
1901     }
1902
1903     my $vmlist = get_vmlist();
1904     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
1905         $error->("this host already contains virtual guests");
1906     }
1907
1908     if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
1909         $error->("corosync is already running, is this node already in a cluster?!");
1910     }
1911
1912     # check if corosync ring IPs are configured on the current nodes interfaces
1913     my $check_ip = sub {
1914         my $ip = shift // return;
1915         my $logid = shift;
1916         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
1917             my $host = $ip;
1918             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
1919             if ($@) {
1920                 $error->("$logid: cannot use '$host': $@\n") ;
1921                 return;
1922             }
1923         }
1924
1925         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
1926         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
1927
1928         $error->("$logid: cannot use IP '$ip', it must be configured exactly once on local node!\n")
1929             if (scalar(@$configured_ips) != 1);
1930     };
1931
1932     $check_ip->($local_addr, 'local node address');
1933     $check_ip->($link0->{address}, 'ring0') if defined($link0);
1934     $check_ip->($link1->{address}, 'ring1') if defined($link1);
1935
1936     if ($errors) {
1937         warn "detected the following error(s):\n$errors";
1938         die "Check if node may join a cluster failed!\n" if !$force;
1939     }
1940 }
1941
1942 # NOTE: filesystem must be offline here, no DB changes allowed
1943 my $backup_cfs_database = sub {
1944     my ($dbfile) = @_;
1945
1946     mkdir $dbbackupdir;
1947
1948     my $ctime = time();
1949     my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
1950
1951     print "backup old database to '$backup_fn'\n";
1952
1953     my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \ ">${backup_fn}"] ];
1954     run_command($cmd, 'errmsg' => "cannot backup old database\n");
1955
1956     my $maxfiles = 10; # purge older backup
1957     my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql.gz> ];
1958
1959     if ((my $count = scalar(@$backups)) > $maxfiles) {
1960         foreach my $f (@$backups[$maxfiles..$count-1]) {
1961             next if $f !~ m/^(\S+)$/; # untaint
1962             print "delete old backup '$1'\n";
1963             unlink $1;
1964         }
1965     }
1966 };
1967
1968 sub join {
1969     my ($param) = @_;
1970
1971     my $nodename = PVE::INotify::nodename();
1972     my $local_ip_address = remote_node_ip($nodename);
1973
1974     my $link0 = parse_corosync_link($param->{link0});
1975     my $link1 = parse_corosync_link($param->{link1});
1976
1977     # check if we can join with the given parameters and current node state
1978     assert_joinable($local_ip_address, $link0, $link1, $param->{force});
1979
1980     setup_sshd_config();
1981     setup_rootsshconfig();
1982     setup_ssh_keys();
1983
1984     # make sure known_hosts is on local filesystem
1985     ssh_unmerge_known_hosts();
1986
1987     my $host = $param->{hostname};
1988     my $conn_args = {
1989         username => 'root@pam',
1990         password => $param->{password},
1991         cookie_name => 'PVEAuthCookie',
1992         protocol => 'https',
1993         host => $host,
1994         port => 8006,
1995     };
1996
1997     if (my $fp = $param->{fingerprint}) {
1998         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
1999     } else {
2000         # API schema ensures that we can only get here from CLI handler
2001         $conn_args->{manual_verification} = 1;
2002     }
2003
2004     print "Establishing API connection with host '$host'\n";
2005
2006     my $conn = PVE::APIClient::LWP->new(%$conn_args);
2007     $conn->login();
2008
2009     # login raises an exception on failure, so if we get here we're good
2010     print "Login succeeded.\n";
2011
2012     my $args = {};
2013     $args->{force} = $param->{force} if defined($param->{force});
2014     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
2015     $args->{votes} = $param->{votes} if defined($param->{votes});
2016     # just pass the un-parsed string through, or as we've address as the
2017     # default_key, we can just pass the fallback directly too
2018     $args->{link0} = $param->{link0} // $local_ip_address;
2019     $args->{link1} = $param->{link1} if defined($param->{link1});
2020
2021     print "Request addition of this node\n";
2022     my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
2023
2024     print "Join request OK, finishing setup locally\n";
2025
2026     # added successfuly - now prepare local node
2027     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
2028 }
2029
2030 sub finish_join {
2031     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
2032
2033     mkdir "$localclusterdir";
2034     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
2035     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
2036
2037     print "stopping pve-cluster service\n";
2038     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
2039     run_command($cmd, errmsg => "can't stop pve-cluster service");
2040
2041     $backup_cfs_database->($dbfile);
2042     unlink $dbfile;
2043
2044     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
2045     run_command($cmd, errmsg => "starting pve-cluster failed");
2046
2047     # wait for quorum
2048     my $printqmsg = 1;
2049     while (!check_cfs_quorum(1)) {
2050         if ($printqmsg) {
2051             print "waiting for quorum...";
2052             STDOUT->flush();
2053             $printqmsg = 0;
2054         }
2055         sleep(1);
2056     }
2057     print "OK\n" if !$printqmsg;
2058
2059     updatecerts_and_ssh(1);
2060
2061     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
2062     run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
2063
2064     print "successfully added node '$nodename' to cluster.\n";
2065 }
2066
2067 sub updatecerts_and_ssh {
2068     my ($force_new_cert, $silent) = @_;
2069
2070     my $p = sub { print "$_[0]\n" if !$silent };
2071
2072     setup_rootsshconfig();
2073
2074     gen_pve_vzdump_symlink();
2075
2076     if (!check_cfs_quorum(1)) {
2077         return undef if $silent;
2078         die "no quorum - unable to update files\n";
2079     }
2080
2081     setup_ssh_keys();
2082
2083     my $nodename = PVE::INotify::nodename();
2084     my $local_ip_address = remote_node_ip($nodename);
2085
2086     $p->("(re)generate node files");
2087     $p->("generate new node certificate") if $force_new_cert;
2088     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
2089
2090     $p->("merge authorized SSH keys and known hosts");
2091     ssh_merge_keys();
2092     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
2093     gen_pve_vzdump_files();
2094 }
2095
2096 1;