data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Digest::HMAC_SHA1;
   7 use Digest::SHA;
   8 use Encode;
   9 use File::stat qw();
  10 use IO::File;
  11 use JSON;
  12 use MIME::Base64;
  13 use Net::SSLeay;
  14 use POSIX qw(EEXIST ENOENT);
  15 use RRDs;
  16 use Socket;
  17 use Storable qw(dclone);
  18 use UUID;
  19
  20 use PVE::INotify;
  21 use PVE::IPCC;
  22 use PVE::JSONSchema;
  23 use PVE::Network;
  24 use PVE::SafeSyslog;
  25 use PVE::Tools qw(run_command);
  26
  27 use PVE::Cluster::IPCConst;
  28
  29 use base 'Exporter';
  30
  31 our @EXPORT_OK = qw(
  32 cfs_read_file
  33 cfs_write_file
  34 cfs_register_file
  35 cfs_lock_file);
  36
  37 use Data::Dumper; # fixme: remove
  38
  39 # x509 certificate utils
  40
  41 my $basedir = "/etc/pve";
  42 my $authdir = "$basedir/priv";
  43 my $lockdir = "/etc/pve/priv/lock";
  44
  45 # cfs and corosync files
  46 my $dbfile = "/var/lib/pve-cluster/config.db";
  47 my $dbbackupdir = "/var/lib/pve-cluster/backup";
  48 my $localclusterdir = "/etc/corosync";
  49 my $localclusterconf = "$localclusterdir/corosync.conf";
  50 my $authfile = "$localclusterdir/authkey";
  51 my $clusterconf = "$basedir/corosync.conf";
  52
  53 my $authprivkeyfn = "$authdir/authkey.key";
  54 my $authpubkeyfn = "$basedir/authkey.pub";
  55 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  56 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  57 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  58 # this is just a secret accessable by the web browser
  59 # and is used for CSRF prevention
  60 my $pvewww_key_fn = "$basedir/pve-www.key";
  61
  62 # ssh related files
  63 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  64 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  65 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  66 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  67 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  68 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  69 my $sshd_config_fn = "/etc/ssh/sshd_config";
  70 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  71 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  72 my $rootsshconfig = "/root/.ssh/config";
  73
  74 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
  75 # observed files are the one we can get directly through IPCC, they are cached
  76 # using a computed version and only those can be used by the cfs_*_file methods
  77 my $observed = {
  78     'vzdump.cron' => 1,
  79     'storage.cfg' => 1,
  80     'datacenter.cfg' => 1,
  81     'replication.cfg' => 1,
  82     'corosync.conf' => 1,
  83     'corosync.conf.new' => 1,
  84     'user.cfg' => 1,
  85     'domains.cfg' => 1,
  86     'priv/shadow.cfg' => 1,
  87     'priv/tfa.cfg' => 1,
  88     '/qemu-server/' => 1,
  89     '/openvz/' => 1,
  90     '/lxc/' => 1,
  91     'ha/crm_commands' => 1,
  92     'ha/manager_status' => 1,
  93     'ha/resources.cfg' => 1,
  94     'ha/groups.cfg' => 1,
  95     'ha/fence.cfg' => 1,
  96     'status.cfg' => 1,
  97     'ceph.conf' => 1,
  98     'sdn.cfg' => 1,
  99     'sdn.cfg.new' => 1,
 100 };
 101
 102 # only write output if something fails
 103 sub run_silent_cmd {
 104     my ($cmd) = @_;
 105
 106     my $outbuf = '';
 107     my $record = sub { $outbuf .= shift . "\n"; };
 108
 109     eval { run_command($cmd, outfunc => $record, errfunc => $record) };
 110
 111     if (my $err = $@) {
 112         print STDERR $outbuf;
 113         die $err;
 114     }
 115 }
 116
 117 sub check_cfs_quorum {
 118     my ($noerr) = @_;
 119
 120     # note: -w filename always return 1 for root, so wee need
 121     # to use File::lstat here
 122     my $st = File::stat::lstat("$basedir/local");
 123     my $quorate = ($st && (($st->mode & 0200) != 0));
 124
 125     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 126
 127     return $quorate;
 128 }
 129
 130 sub check_cfs_is_mounted {
 131     my ($noerr) = @_;
 132
 133     my $res = -l "$basedir/local";
 134
 135     die "pve configuration filesystem not mounted\n"
 136         if !$res && !$noerr;
 137
 138     return $res;
 139 }
 140
 141 sub gen_local_dirs {
 142     my ($nodename) = @_;
 143
 144     check_cfs_is_mounted();
 145
 146     my @required_dirs = (
 147         "$basedir/priv",
 148         "$basedir/nodes",
 149         "$basedir/nodes/$nodename",
 150         "$basedir/nodes/$nodename/lxc",
 151         "$basedir/nodes/$nodename/qemu-server",
 152         "$basedir/nodes/$nodename/openvz",
 153         "$basedir/nodes/$nodename/priv");
 154
 155     foreach my $dir (@required_dirs) {
 156         if (! -d $dir) {
 157             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 158         }
 159     }
 160 }
 161
 162 sub gen_auth_key {
 163
 164     return if -f "$authprivkeyfn";
 165
 166     check_cfs_is_mounted();
 167
 168     cfs_lock_authkey(undef, sub {
 169         mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
 170
 171         run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 172
 173         run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 174     });
 175
 176     die "$@\n" if $@;
 177 }
 178
 179 sub gen_pveca_key {
 180
 181     return if -f $pveca_key_fn;
 182
 183     eval {
 184         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 185     };
 186
 187     die "unable to generate pve ca key:\n$@" if $@;
 188 }
 189
 190 sub gen_pveca_cert {
 191
 192     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 193         return 0;
 194     }
 195
 196     gen_pveca_key();
 197
 198     # we try to generate an unique 'subject' to avoid browser problems
 199     # (reused serial numbers, ..)
 200     my $uuid;
 201     UUID::generate($uuid);
 202     my $uuid_str;
 203     UUID::unparse($uuid, $uuid_str);
 204
 205     eval {
 206         # wrap openssl with faketime to prevent bug #904
 207         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 208                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 209                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 210                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 211     };
 212
 213     die "generating pve root certificate failed:\n$@" if $@;
 214
 215     return 1;
 216 }
 217
 218 sub gen_pve_ssl_key {
 219     my ($nodename) = @_;
 220
 221     die "no node name specified" if !$nodename;
 222
 223     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 224
 225     return if -f $pvessl_key_fn;
 226
 227     eval {
 228         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 229     };
 230
 231     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 232 }
 233
 234 sub gen_pve_www_key {
 235
 236     return if -f $pvewww_key_fn;
 237
 238     eval {
 239         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 240     };
 241
 242     die "unable to generate pve www key:\n$@" if $@;
 243 }
 244
 245 sub update_serial {
 246     my ($serial) = @_;
 247
 248     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 249 }
 250
 251 sub gen_pve_ssl_cert {
 252     my ($force, $nodename, $ip) = @_;
 253
 254     die "no node name specified" if !$nodename;
 255     die "no IP specified" if !$ip;
 256
 257     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 258
 259     return if !$force && -f $pvessl_cert_fn;
 260
 261     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 262
 263     my $rc = PVE::INotify::read_file('resolvconf');
 264
 265     $names .= ",IP:$ip";
 266
 267     my $fqdn = $nodename;
 268
 269     $names .= ",DNS:$nodename";
 270
 271     if ($rc && $rc->{search}) {
 272         $fqdn = $nodename . "." . $rc->{search};
 273         $names .= ",DNS:$fqdn";
 274     }
 275
 276     my $sslconf = <<__EOD;
 277 RANDFILE = /root/.rnd
 278 extensions = v3_req
 279
 280 [ req ]
 281 default_bits = 2048
 282 distinguished_name = req_distinguished_name
 283 req_extensions = v3_req
 284 prompt = no
 285 string_mask = nombstr
 286
 287 [ req_distinguished_name ]
 288 organizationalUnitName = PVE Cluster Node
 289 organizationName = Proxmox Virtual Environment
 290 commonName = $fqdn
 291
 292 [ v3_req ]
 293 basicConstraints = CA:FALSE
 294 extendedKeyUsage = serverAuth
 295 subjectAltName = $names
 296 __EOD
 297
 298     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 299     my $fh = IO::File->new ($cfgfn, "w");
 300     print $fh $sslconf;
 301     close ($fh);
 302
 303     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 304     unlink $reqfn;
 305
 306     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 307     eval {
 308         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 309                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 310     };
 311
 312     if (my $err = $@) {
 313         unlink $reqfn;
 314         unlink $cfgfn;
 315         die "unable to generate pve certificate request:\n$err";
 316     }
 317
 318     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 319
 320     eval {
 321         # wrap openssl with faketime to prevent bug #904
 322         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
 323                         '-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
 324                         '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
 325                         '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
 326     };
 327
 328     if (my $err = $@) {
 329         unlink $reqfn;
 330         unlink $cfgfn;
 331         die "unable to generate pve ssl certificate:\n$err";
 332     }
 333
 334     unlink $cfgfn;
 335     unlink $reqfn;
 336 }
 337
 338 sub gen_pve_node_files {
 339     my ($nodename, $ip, $opt_force) = @_;
 340
 341     gen_local_dirs($nodename);
 342
 343     gen_auth_key();
 344
 345     # make sure we have a (cluster wide) secret
 346     # for CSRFR prevention
 347     gen_pve_www_key();
 348
 349     # make sure we have a (per node) private key
 350     gen_pve_ssl_key($nodename);
 351
 352     # make sure we have a CA
 353     my $force = gen_pveca_cert();
 354
 355     $force = 1 if $opt_force;
 356
 357     gen_pve_ssl_cert($force, $nodename, $ip);
 358 }
 359
 360 my $vzdump_cron_dummy = <<__EOD;
 361 # cluster wide vzdump cron schedule
 362 # Atomatically generated file - do not edit
 363
 364 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 365
 366 __EOD
 367
 368 sub gen_pve_vzdump_symlink {
 369
 370     my $filename = "/etc/pve/vzdump.cron";
 371
 372     my $link_fn = "/etc/cron.d/vzdump";
 373
 374     if ((-f $filename) && (! -l $link_fn)) {
 375         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 376         symlink($filename, $link_fn);
 377     }
 378 }
 379
 380 sub gen_pve_vzdump_files {
 381
 382     my $filename = "/etc/pve/vzdump.cron";
 383
 384     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 385         if ! -f $filename;
 386
 387     gen_pve_vzdump_symlink();
 388 };
 389
 390 my $versions = {};
 391 my $vmlist = {};
 392 my $clinfo = {};
 393
 394 my $ipcc_send_rec = sub {
 395     my ($msgid, $data) = @_;
 396
 397     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 398
 399     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 400
 401     return $res;
 402 };
 403
 404 my $ipcc_send_rec_json = sub {
 405     my ($msgid, $data) = @_;
 406
 407     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 408
 409     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 410
 411     return decode_json($res);
 412 };
 413
 414 my $ipcc_get_config = sub {
 415     my ($path) = @_;
 416
 417     my $bindata = pack "Z*", $path;
 418     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_CONFIG, $bindata);
 419     if (!defined($res)) {
 420         if ($! != 0) {
 421             return undef if $! == ENOENT;
 422             die "$!\n";
 423         }
 424         return '';
 425     }
 426
 427     return $res;
 428 };
 429
 430 my $ipcc_get_status = sub {
 431     my ($name, $nodename) = @_;
 432
 433     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 434     return PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_STATUS, $bindata);
 435 };
 436
 437 my $ipcc_remove_status = sub {
 438     my ($name) = @_;
 439     # we just omit the data payload, pmxcfs takes this as hint and removes this
 440     # key from the status hashtable
 441     my $bindata = pack "Z[256]", $name;
 442     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 443 };
 444
 445 my $ipcc_update_status = sub {
 446     my ($name, $data) = @_;
 447
 448     my $raw = ref($data) ? encode_json($data) : $data;
 449     # update status
 450     my $bindata = pack "Z[256]Z*", $name, $raw;
 451
 452     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 453 };
 454
 455 my $ipcc_log = sub {
 456     my ($priority, $ident, $tag, $msg) = @_;
 457
 458     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 459     bytes::length($tag) + 1, $ident, $tag, $msg;
 460
 461     return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG, $bindata);
 462 };
 463
 464 my $ipcc_get_cluster_log = sub {
 465     my ($user, $max) = @_;
 466
 467     $max = 0 if !defined($max);
 468
 469     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 470     return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG, $bindata);
 471 };
 472
 473 my $ccache = {};
 474
 475 sub cfs_update {
 476     my ($fail) = @_;
 477     eval {
 478         my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION);
 479         #warn "GOT1: " . Dumper($res);
 480         die "no starttime\n" if !$res->{starttime};
 481
 482         if (!$res->{starttime} || !$versions->{starttime} ||
 483             $res->{starttime} != $versions->{starttime}) {
 484             #print "detected changed starttime\n";
 485             $vmlist = {};
 486             $clinfo = {};
 487             $ccache = {};
 488         }
 489
 490         $versions = $res;
 491     };
 492     my $err = $@;
 493     if ($err) {
 494         $versions = {};
 495         $vmlist = {};
 496         $clinfo = {};
 497         $ccache = {};
 498         die $err if $fail;
 499         warn $err;
 500     }
 501
 502     eval {
 503         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 504             #warn "detected new clinfo\n";
 505             $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO);
 506         }
 507     };
 508     $err = $@;
 509     if ($err) {
 510         $clinfo = {};
 511         die $err if $fail;
 512         warn $err;
 513     }
 514
 515     eval {
 516         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 517             #warn "detected new vmlist1\n";
 518             $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST);
 519         }
 520     };
 521     $err = $@;
 522     if ($err) {
 523         $vmlist = {};
 524         die $err if $fail;
 525         warn $err;
 526     }
 527 }
 528
 529 sub get_vmlist {
 530     return $vmlist;
 531 }
 532
 533 sub get_clinfo {
 534     return $clinfo;
 535 }
 536
 537 sub get_members {
 538     return $clinfo->{nodelist};
 539 }
 540
 541 sub get_nodelist {
 542     my $nodelist = $clinfo->{nodelist};
 543
 544     my $nodename = PVE::INotify::nodename();
 545
 546     if (!$nodelist || !$nodelist->{$nodename}) {
 547         return [ $nodename ];
 548     }
 549
 550     return [ keys %$nodelist ];
 551 }
 552
 553 # only stored in a in-memory hashtable inside pmxcfs, local data is gone after
 554 # a restart (of pmxcfs or the node), peer data is still available then
 555 # best used for status data, like running (ceph) services, package versions, ...
 556 sub broadcast_node_kv {
 557     my ($key, $data) = @_;
 558
 559     if (!defined($data)) {
 560         eval {
 561             $ipcc_remove_status->("kv/$key");
 562         };
 563     } else {
 564         die "cannot send a reference\n" if ref($data);
 565         my $size = length($data);
 566         die "data for '$key' too big\n" if $size >= (32 * 1024); # limit from pmxfs
 567
 568         eval {
 569             $ipcc_update_status->("kv/$key", $data);
 570         };
 571     }
 572
 573     warn $@ if $@;
 574 }
 575
 576 # nodename is optional
 577 sub get_node_kv {
 578     my ($key, $nodename) = @_;
 579
 580     my $res = {};
 581     my $get_node_data = sub {
 582         my ($node) = @_;
 583         my $raw = $ipcc_get_status->("kv/$key", $node);
 584         $res->{$node} = unpack("Z*", $raw) if $raw;
 585     };
 586
 587     if ($nodename) {
 588         $get_node_data->($nodename);
 589     } else {
 590         my $nodelist = get_nodelist();
 591
 592         foreach my $node (@$nodelist) {
 593             $get_node_data->($node);
 594         }
 595     }
 596
 597     return $res;
 598 }
 599
 600 # property: a config property you want to get, e.g., this is perfect to get
 601 # the 'lock' entry of a guest _fast_ (>100 faster than manual parsing here)
 602 # vmid: optipnal, if a valid is passed we only check that one, else return all
 603 # NOTE: does *not* searches snapshot and PENDING entries sections!
 604 sub get_guest_config_property {
 605     my ($property, $vmid) = @_;
 606
 607     die "property is required" if !defined($property);
 608
 609     my $bindata = pack "VZ*", $vmid // 0, $property;
 610     my $res = $ipcc_send_rec_json->(CFS_IPC_GET_GUEST_CONFIG_PROPERTY, $bindata);
 611
 612     return $res;
 613 }
 614
 615 # $data must be a chronological descending ordered array of tasks
 616 sub broadcast_tasklist {
 617     my ($data) = @_;
 618
 619     # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
 620     # from pmxcfs) - drop older items until we satisfy this constraint
 621     my $size = length(encode_json($data));
 622     while ($size >= (32 * 1024)) {
 623         pop @$data;
 624         $size = length(encode_json($data));
 625     }
 626
 627     eval {
 628         &$ipcc_update_status("tasklist", $data);
 629     };
 630
 631     warn $@ if $@;
 632 }
 633
 634 my $tasklistcache = {};
 635
 636 sub get_tasklist {
 637     my ($nodename) = @_;
 638
 639     my $kvstore = $versions->{kvstore} || {};
 640
 641     my $nodelist = get_nodelist();
 642
 643     my $res = [];
 644     foreach my $node (@$nodelist) {
 645         next if $nodename && ($nodename ne $node);
 646         eval {
 647             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 648             my $cd = $tasklistcache->{$node};
 649             if (!$cd || !$ver || !$cd->{version} ||
 650                 ($cd->{version} != $ver)) {
 651                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 652                 my $data = decode_json($raw);
 653                 push @$res, @$data;
 654                 $cd = $tasklistcache->{$node} = {
 655                     data => $data,
 656                     version => $ver,
 657                 };
 658             } elsif ($cd && $cd->{data}) {
 659                 push @$res, @{$cd->{data}};
 660             }
 661         };
 662         my $err = $@;
 663         syslog('err', $err) if $err;
 664     }
 665
 666     return $res;
 667 }
 668
 669 sub broadcast_rrd {
 670     my ($rrdid, $data) = @_;
 671
 672     eval {
 673         &$ipcc_update_status("rrd/$rrdid", $data);
 674     };
 675     my $err = $@;
 676
 677     warn $err if $err;
 678 }
 679
 680 my $last_rrd_dump = 0;
 681 my $last_rrd_data = "";
 682
 683 sub rrd_dump {
 684
 685     my $ctime = time();
 686
 687     my $diff = $ctime - $last_rrd_dump;
 688     if ($diff < 2) {
 689         return $last_rrd_data;
 690     }
 691
 692     my $raw;
 693     eval {
 694         $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP);
 695     };
 696     my $err = $@;
 697
 698     if ($err) {
 699         warn $err;
 700         return {};
 701     }
 702
 703     my $res = {};
 704
 705     if ($raw) {
 706         while ($raw =~ s/^(.*)\n//) {
 707             my ($key, @ela) = split(/:/, $1);
 708             next if !$key;
 709             next if !(scalar(@ela) > 1);
 710             $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
 711         }
 712     }
 713
 714     $last_rrd_dump = $ctime;
 715     $last_rrd_data = $res;
 716
 717     return $res;
 718 }
 719
 720 sub create_rrd_data {
 721     my ($rrdname, $timeframe, $cf) = @_;
 722
 723     my $rrddir = "/var/lib/rrdcached/db";
 724
 725     my $rrd = "$rrddir/$rrdname";
 726
 727     my $setup = {
 728         hour =>  [ 60, 70 ],
 729         day  =>  [ 60*30, 70 ],
 730         week =>  [ 60*180, 70 ],
 731         month => [ 60*720, 70 ],
 732         year =>  [ 60*10080, 70 ],
 733     };
 734
 735     my ($reso, $count) = @{$setup->{$timeframe}};
 736     my $ctime  = $reso*int(time()/$reso);
 737     my $req_start = $ctime - $reso*$count;
 738
 739     $cf = "AVERAGE" if !$cf;
 740
 741     my @args = (
 742         "-s" => $req_start,
 743         "-e" => $ctime - 1,
 744         "-r" => $reso,
 745         );
 746
 747     my $socket = "/var/run/rrdcached.sock";
 748     push @args, "--daemon" => "unix:$socket" if -S $socket;
 749
 750     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 751
 752     my $err = RRDs::error;
 753     die "RRD error: $err\n" if $err;
 754
 755     die "got wrong time resolution ($step != $reso)\n"
 756         if $step != $reso;
 757
 758     my $res = [];
 759     my $fields = scalar(@$names);
 760     for my $line (@$data) {
 761         my $entry = { 'time' => $start };
 762         $start += $step;
 763         for (my $i = 0; $i < $fields; $i++) {
 764             my $name = $names->[$i];
 765             if (defined(my $val = $line->[$i])) {
 766                 $entry->{$name} = $val;
 767             } else {
 768                 # leave empty fields undefined
 769                 # maybe make this configurable?
 770             }
 771         }
 772         push @$res, $entry;
 773     }
 774
 775     return $res;
 776 }
 777
 778 sub create_rrd_graph {
 779     my ($rrdname, $timeframe, $ds, $cf) = @_;
 780
 781     # Using RRD graph is clumsy - maybe it
 782     # is better to simply fetch the data, and do all display
 783     # related things with javascript (new extjs html5 graph library).
 784
 785     my $rrddir = "/var/lib/rrdcached/db";
 786
 787     my $rrd = "$rrddir/$rrdname";
 788
 789     my @ids = PVE::Tools::split_list($ds);
 790
 791     my $ds_txt = join('_', @ids);
 792
 793     my $filename = "${rrd}_${ds_txt}.png";
 794
 795     my $setup = {
 796         hour =>  [ 60, 60 ],
 797         day  =>  [ 60*30, 70 ],
 798         week =>  [ 60*180, 70 ],
 799         month => [ 60*720, 70 ],
 800         year =>  [ 60*10080, 70 ],
 801     };
 802
 803     my ($reso, $count) = @{$setup->{$timeframe}};
 804
 805     my @args = (
 806         "--imgformat" => "PNG",
 807         "--border" => 0,
 808         "--height" => 200,
 809         "--width" => 800,
 810         "--start" => - $reso*$count,
 811         "--end" => 'now' ,
 812         "--lower-limit" => 0,
 813         );
 814
 815     my $socket = "/var/run/rrdcached.sock";
 816     push @args, "--daemon" => "unix:$socket" if -S $socket;
 817
 818     my @coldef = ('#00ddff', '#ff0000');
 819
 820     $cf = "AVERAGE" if !$cf;
 821
 822     my $i = 0;
 823     foreach my $id (@ids) {
 824         my $col = $coldef[$i++] || die "fixme: no color definition";
 825         push @args, "DEF:${id}=$rrd:${id}:$cf";
 826         my $dataid = $id;
 827         if ($id eq 'cpu' || $id eq 'iowait') {
 828             push @args, "CDEF:${id}_per=${id},100,*";
 829             $dataid = "${id}_per";
 830         }
 831         push @args, "LINE2:${dataid}${col}:${id}";
 832     }
 833
 834     push @args, '--full-size-mode';
 835
 836     # we do not really store data into the file
 837     my $res = RRDs::graphv('-', @args);
 838
 839     my $err = RRDs::error;
 840     die "RRD error: $err\n" if $err;
 841
 842     return { filename => $filename, image => $res->{image} };
 843 }
 844
 845 # a fast way to read files (avoid fuse overhead)
 846 sub get_config {
 847     my ($path) = @_;
 848
 849     return &$ipcc_get_config($path);
 850 }
 851
 852 sub get_cluster_log {
 853     my ($user, $max) = @_;
 854
 855     return &$ipcc_get_cluster_log($user, $max);
 856 }
 857
 858 my $file_info = {};
 859
 860 sub cfs_register_file {
 861     my ($filename, $parser, $writer) = @_;
 862
 863     $observed->{$filename} || die "unknown file '$filename'";
 864
 865     die "file '$filename' already registered" if $file_info->{$filename};
 866
 867     $file_info->{$filename} = {
 868         parser => $parser,
 869         writer => $writer,
 870     };
 871 }
 872
 873 my $ccache_read = sub {
 874     my ($filename, $parser, $version) = @_;
 875
 876     $ccache->{$filename} = {} if !$ccache->{$filename};
 877
 878     my $ci = $ccache->{$filename};
 879
 880     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 881         # we always call the parser, even when the file does not exists
 882         # (in that case $data is undef)
 883         my $data = get_config($filename);
 884         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 885         $ci->{version} = $version;
 886     }
 887
 888     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 889
 890     return $res;
 891 };
 892
 893 sub cfs_file_version {
 894     my ($filename) = @_;
 895
 896     my $version;
 897     my $infotag;
 898     if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
 899         my ($type, $vmid) = ($1, $2);
 900         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 901             $version = $vmlist->{ids}->{$vmid}->{version};
 902         }
 903         $infotag = "/$type/";
 904     } else {
 905         $infotag = $filename;
 906         $version = $versions->{$filename};
 907     }
 908
 909     my $info = $file_info->{$infotag} ||
 910         die "unknown file type '$filename'\n";
 911
 912     return wantarray ? ($version, $info) : $version;
 913 }
 914
 915 sub cfs_read_file {
 916     my ($filename) = @_;
 917
 918     my ($version, $info) = cfs_file_version($filename);
 919     my $parser = $info->{parser};
 920
 921     return &$ccache_read($filename, $parser, $version);
 922 }
 923
 924 sub cfs_write_file {
 925     my ($filename, $data) = @_;
 926
 927     my ($version, $info) = cfs_file_version($filename);
 928
 929     my $writer = $info->{writer} || die "no writer defined";
 930
 931     my $fsname = "/etc/pve/$filename";
 932
 933     my $raw = &$writer($fsname, $data);
 934
 935     if (my $ci = $ccache->{$filename}) {
 936         $ci->{version} = undef;
 937     }
 938
 939     PVE::Tools::file_set_contents($fsname, $raw);
 940 }
 941
 942 my $cfs_lock = sub {
 943     my ($lockid, $timeout, $code, @param) = @_;
 944
 945     my $prev_alarm = alarm(0); # suspend outer alarm early
 946
 947     my $res;
 948     my $got_lock = 0;
 949
 950     # this timeout is for acquire the lock
 951     $timeout = 10 if !$timeout;
 952
 953     my $filename = "$lockdir/$lockid";
 954
 955     eval {
 956
 957         mkdir $lockdir;
 958
 959         if (! -d $lockdir) {
 960             die "pve cluster filesystem not online.\n";
 961         }
 962
 963         my $timeout_err = sub { die "got lock request timeout\n"; };
 964         local $SIG{ALRM} = $timeout_err;
 965
 966         while (1) {
 967             alarm ($timeout);
 968             $got_lock = mkdir($filename);
 969             $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
 970
 971             last if $got_lock;
 972
 973             $timeout_err->() if $timeout <= 0;
 974
 975             print STDERR "trying to acquire cfs lock '$lockid' ...\n";
 976             utime (0, 0, $filename); # cfs unlock request
 977             sleep(1);
 978         }
 979
 980         # fixed command timeout: cfs locks have a timeout of 120
 981         # using 60 gives us another 60 seconds to abort the task
 982         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 983         alarm(60);
 984
 985         cfs_update(); # make sure we read latest versions inside code()
 986
 987         $res = &$code(@param);
 988
 989         alarm(0);
 990     };
 991
 992     my $err = $@;
 993
 994     $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
 995
 996     rmdir $filename if $got_lock; # if we held the lock always unlock again
 997
 998     alarm($prev_alarm);
 999
1000     if ($err) {
1001         $@ = "error with cfs lock '$lockid': $err";
1002         return undef;
1003     }
1004
1005     $@ = undef;
1006
1007     return $res;
1008 };
1009
1010 sub cfs_lock_file {
1011     my ($filename, $timeout, $code, @param) = @_;
1012
1013     my $info = $observed->{$filename} || die "unknown file '$filename'";
1014
1015     my $lockid = "file-$filename";
1016     $lockid =~ s/[.\/]/_/g;
1017
1018     &$cfs_lock($lockid, $timeout, $code, @param);
1019 }
1020
1021 sub cfs_lock_storage {
1022     my ($storeid, $timeout, $code, @param) = @_;
1023
1024     my $lockid = "storage-$storeid";
1025
1026     &$cfs_lock($lockid, $timeout, $code, @param);
1027 }
1028
1029 sub cfs_lock_domain {
1030     my ($domainname, $timeout, $code, @param) = @_;
1031
1032     my $lockid = "domain-$domainname";
1033
1034     &$cfs_lock($lockid, $timeout, $code, @param);
1035 }
1036
1037 sub cfs_lock_acme {
1038     my ($account, $timeout, $code, @param) = @_;
1039
1040     my $lockid = "acme-$account";
1041
1042     &$cfs_lock($lockid, $timeout, $code, @param);
1043 }
1044
1045 sub cfs_lock_authkey {
1046     my ($timeout, $code, @param) = @_;
1047
1048     $cfs_lock->('authkey', $timeout, $code, @param);
1049 }
1050
1051 my $log_levels = {
1052     "emerg" => 0,
1053     "alert" => 1,
1054     "crit" => 2,
1055     "critical" => 2,
1056     "err" => 3,
1057     "error" => 3,
1058     "warn" => 4,
1059     "warning" => 4,
1060     "notice" => 5,
1061     "info" => 6,
1062     "debug" => 7,
1063 };
1064
1065 sub log_msg {
1066    my ($priority, $ident, $msg) = @_;
1067
1068    if (my $tmp = $log_levels->{$priority}) {
1069        $priority = $tmp;
1070    }
1071
1072    die "need numeric log priority" if $priority !~ /^\d+$/;
1073
1074    my $tag = PVE::SafeSyslog::tag();
1075
1076    $msg = "empty message" if !$msg;
1077
1078    $ident = "" if !$ident;
1079    $ident = encode("ascii", $ident,
1080                    sub { sprintf "\\u%04x", shift });
1081
1082    my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
1083
1084    if ($ident) {
1085        syslog($priority, "<%s> %s", $ident, $ascii);
1086    } else {
1087        syslog($priority, "%s", $ascii);
1088    }
1089
1090    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
1091
1092    syslog("err", "writing cluster log failed: $@") if $@;
1093 }
1094
1095 sub check_vmid_unused {
1096     my ($vmid, $noerr) = @_;
1097
1098     my $vmlist = get_vmlist();
1099
1100     my $d = $vmlist->{ids}->{$vmid};
1101     return 1 if !defined($d);
1102
1103     return undef if $noerr;
1104
1105     my $vmtypestr =  $d->{type} eq 'qemu' ? 'VM' : 'CT';
1106     die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
1107 }
1108
1109 sub check_node_exists {
1110     my ($nodename, $noerr) = @_;
1111
1112     my $nodelist = $clinfo->{nodelist};
1113     return 1 if $nodelist && $nodelist->{$nodename};
1114
1115     return undef if $noerr;
1116
1117     die "no such cluster node '$nodename'\n";
1118 }
1119
1120 # this is also used to get the IP of the local node
1121 sub remote_node_ip {
1122     my ($nodename, $noerr) = @_;
1123
1124     my $nodelist = $clinfo->{nodelist};
1125     if ($nodelist && $nodelist->{$nodename}) {
1126         if (my $ip = $nodelist->{$nodename}->{ip}) {
1127             return $ip if !wantarray;
1128             my $family = $nodelist->{$nodename}->{address_family};
1129             if (!$family) {
1130                 $nodelist->{$nodename}->{address_family} =
1131                     $family =
1132                     PVE::Tools::get_host_address_family($ip);
1133             }
1134             return wantarray ? ($ip, $family) : $ip;
1135         }
1136     }
1137
1138     # fallback: try to get IP by other means
1139     return PVE::Network::get_ip_from_hostname($nodename, $noerr);
1140 }
1141
1142 sub get_local_migration_ip {
1143     my ($migration_network, $noerr) = @_;
1144
1145     my $cidr = $migration_network;
1146
1147     if (!defined($cidr)) {
1148         my $dc_conf = cfs_read_file('datacenter.cfg');
1149         $cidr = $dc_conf->{migration}->{network}
1150           if defined($dc_conf->{migration}->{network});
1151     }
1152
1153     if (defined($cidr)) {
1154         my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
1155
1156         die "could not get migration ip: no IP address configured on local " .
1157             "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
1158
1159         die "could not get migration ip: multiple IP address configured for " .
1160             "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
1161
1162         return @$ips[0];
1163     }
1164
1165     return undef;
1166 };
1167
1168 # ssh related utility functions
1169
1170 sub ssh_merge_keys {
1171     # remove duplicate keys in $sshauthkeys
1172     # ssh-copy-id simply add keys, so the file can grow to large
1173
1174     my $data = '';
1175     if (-f $sshauthkeys) {
1176         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
1177         chomp($data);
1178     }
1179
1180     my $found_backup;
1181     if (-f $rootsshauthkeysbackup) {
1182         $data .= "\n";
1183         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
1184         chomp($data);
1185         $found_backup = 1;
1186     }
1187
1188     # always add ourself
1189     if (-f $ssh_rsa_id) {
1190         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
1191         chomp($pub);
1192         $data .= "\n$pub\n";
1193     }
1194
1195     my $newdata = "";
1196     my $vhash = {};
1197     my @lines = split(/\n/, $data);
1198     foreach my $line (@lines) {
1199         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1200             next if $vhash->{$3}++;
1201         }
1202         $newdata .= "$line\n";
1203     }
1204
1205     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
1206
1207     if ($found_backup && -l $rootsshauthkeys) {
1208         # everything went well, so we can remove the backup
1209         unlink $rootsshauthkeysbackup;
1210     }
1211 }
1212
1213 sub setup_sshd_config {
1214     my () = @_;
1215
1216     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
1217
1218     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1219
1220     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1221         chomp $conf;
1222         $conf .= "\nPermitRootLogin yes\n";
1223     }
1224
1225     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
1226
1227     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
1228 }
1229
1230 sub setup_rootsshconfig {
1231
1232     # create ssh key if it does not exist
1233     if (! -f $ssh_rsa_id) {
1234         mkdir '/root/.ssh/';
1235         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1236     }
1237
1238     # create ssh config if it does not exist
1239     if (! -f $rootsshconfig) {
1240         mkdir '/root/.ssh';
1241         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
1242             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
1243             # changed order to put AES before Chacha20 (most hardware has AESNI)
1244             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
1245             close($fh);
1246         }
1247     }
1248 }
1249
1250 sub setup_ssh_keys {
1251
1252     mkdir $authdir;
1253
1254     my $import_ok;
1255
1256     if (! -f $sshauthkeys) {
1257         my $old;
1258         if (-f $rootsshauthkeys) {
1259             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
1260         }
1261         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1262             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
1263             close($fh);
1264             $import_ok = 1;
1265         }
1266     }
1267
1268     warn "can't create shared ssh key database '$sshauthkeys'\n"
1269         if ! -f $sshauthkeys;
1270
1271     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
1272         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1273             warn "rename $rootsshauthkeys failed - $!\n";
1274         }
1275     }
1276
1277     if (! -l $rootsshauthkeys) {
1278         symlink $sshauthkeys, $rootsshauthkeys;
1279     }
1280
1281     if (! -l $rootsshauthkeys) {
1282         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1283     } else {
1284         unlink $rootsshauthkeysbackup if $import_ok;
1285     }
1286 }
1287
1288 sub ssh_unmerge_known_hosts {
1289     return if ! -l $sshglobalknownhosts;
1290
1291     my $old = '';
1292     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1293         if -f $sshknownhosts;
1294
1295     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1296 }
1297
1298 sub ssh_merge_known_hosts {
1299     my ($nodename, $ip_address, $createLink) = @_;
1300
1301     die "no node name specified" if !$nodename;
1302     die "no ip address specified" if !$ip_address;
1303
1304     # ssh lowercases hostnames (aliases) before comparision, so we need too
1305     $nodename = lc($nodename);
1306     $ip_address = lc($ip_address);
1307
1308     mkdir $authdir;
1309
1310     if (! -f $sshknownhosts) {
1311         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1312             close($fh);
1313         }
1314     }
1315
1316     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1317
1318     my $new = '';
1319
1320     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1321         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1322     }
1323
1324     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1325     # Note: file sometimes containe emty lines at start, so we use multiline match
1326     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
1327     $hostkey = $1;
1328
1329     my $data = '';
1330     my $vhash = {};
1331
1332     my $found_nodename;
1333     my $found_local_ip;
1334
1335     my $merge_line = sub {
1336         my ($line, $all) = @_;
1337
1338         return if $line =~ m/^\s*$/; # skip empty lines
1339         return if $line =~ m/^#/; # skip comments
1340
1341         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1342             my $key = $1;
1343             my $rsakey = $2;
1344             if (!$vhash->{$key}) {
1345                 $vhash->{$key} = 1;
1346                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1347                     my $salt = decode_base64($1);
1348                     my $digest = $2;
1349                     my $hmac = Digest::HMAC_SHA1->new($salt);
1350                     $hmac->add($nodename);
1351                     my $hd = $hmac->b64digest . '=';
1352                     if ($digest eq $hd) {
1353                         if ($rsakey eq $hostkey) {
1354                             $found_nodename = 1;
1355                             $data .= $line;
1356                         }
1357                         return;
1358                     }
1359                     $hmac = Digest::HMAC_SHA1->new($salt);
1360                     $hmac->add($ip_address);
1361                     $hd = $hmac->b64digest . '=';
1362                     if ($digest eq $hd) {
1363                         if ($rsakey eq $hostkey) {
1364                             $found_local_ip = 1;
1365                             $data .= $line;
1366                         }
1367                         return;
1368                     }
1369                 } else {
1370                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
1371                     if ($key eq $ip_address) {
1372                         $found_local_ip = 1 if $rsakey eq $hostkey;
1373                     } elsif ($key eq $nodename) {
1374                         $found_nodename = 1 if $rsakey eq $hostkey;
1375                     }
1376                 }
1377                 $data .= $line;
1378             }
1379         } elsif ($all) {
1380             $data .= $line;
1381         }
1382     };
1383
1384     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1385         my $line = "$2\n";
1386         &$merge_line($line, 1);
1387     }
1388
1389     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1390         my $line = "$2\n";
1391         &$merge_line($line);
1392     }
1393
1394     # add our own key if not already there
1395     $data .= "$nodename $hostkey\n" if !$found_nodename;
1396     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
1397
1398     PVE::Tools::file_set_contents($sshknownhosts, $data);
1399
1400     return if !$createLink;
1401
1402     unlink $sshglobalknownhosts;
1403     symlink $sshknownhosts, $sshglobalknownhosts;
1404
1405     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1406         if ! -l $sshglobalknownhosts;
1407
1408 }
1409
1410 my $migration_format = {
1411     type => {
1412         default_key => 1,
1413         type => 'string',
1414         enum => ['secure', 'insecure'],
1415         description => "Migration traffic is encrypted using an SSH tunnel by " .
1416           "default. On secure, completely private networks this can be " .
1417           "disabled to increase performance.",
1418         default => 'secure',
1419     },
1420     network => {
1421         optional => 1,
1422         type => 'string', format => 'CIDR',
1423         format_description => 'CIDR',
1424         description => "CIDR of the (sub) network that is used for migration."
1425     },
1426 };
1427
1428 my $ha_format = {
1429     shutdown_policy => {
1430         type => 'string',
1431         enum => ['freeze', 'failover', 'conditional'],
1432         description => "The policy for HA services on node shutdown. 'freeze' disables auto-recovery, 'failover' ensures recovery, 'conditional' recovers on poweroff and freezes on reboot. Running HA Services will always get stopped first on shutdown.",
1433         verbose_description => "Describes the policy for handling HA services on poweroff or reboot of a node. Freeze will always freeze services which are still located on the node on shutdown, those services won't be recovered by the HA manager. Failover will not mark the services as frozen and thus the services will get recovered to other nodes, if the shutdown node does not come up again quickly (< 1min). 'conditional' chooses automatically depending on the type of shutdown, i.e., on a reboot the service will be frozen but on a poweroff the service will stay as is, and thus get recovered after about 2 minutes.",
1434         default => 'conditional',
1435     }
1436 };
1437
1438 PVE::JSONSchema::register_format('mac-prefix', \&pve_verify_mac_prefix);
1439 sub pve_verify_mac_prefix {
1440     my ($mac_prefix, $noerr) = @_;
1441
1442     if ($mac_prefix !~ m/^[a-f0-9][02468ace](?::[a-f0-9]{2}){0,2}:?$/i) {
1443         return undef if $noerr;
1444         die "value is not a valid unicast MAC address prefix\n";
1445     }
1446     return $mac_prefix;
1447 }
1448
1449 our $u2f_format = {
1450     appid => {
1451         type => 'string',
1452         description => "U2F AppId URL override. Defaults to the origin.",
1453         format_description => 'APPID',
1454         optional => 1,
1455     },
1456     origin => {
1457         type => 'string',
1458         description => "U2F Origin override. Mostly useful for single nodes with a single URL.",
1459         format_description => 'URL',
1460         optional => 1,
1461     },
1462 };
1463
1464 my $datacenter_schema = {
1465     type => "object",
1466     additionalProperties => 0,
1467     properties => {
1468         keyboard => {
1469             optional => 1,
1470             type => 'string',
1471             description => "Default keybord layout for vnc server.",
1472             enum => PVE::Tools::kvmkeymaplist(),
1473         },
1474         language => {
1475             optional => 1,
1476             type => 'string',
1477             description => "Default GUI language.",
1478             enum => [
1479                 'zh_CN',
1480                 'zh_TW',
1481                 'ca',
1482                 'en',
1483                 'eu',
1484                 'fr',
1485                 'de',
1486                 'it',
1487                 'es',
1488                 'ja',
1489                 'nb',
1490                 'nn',
1491                 'fa',
1492                 'pl',
1493                 'pt_BR',
1494                 'ru',
1495                 'sl',
1496                 'sv',
1497                 'tr',
1498             ],
1499         },
1500         http_proxy => {
1501             optional => 1,
1502             type => 'string',
1503             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1504             pattern => "http://.*",
1505         },
1506         migration_unsecure => {
1507             optional => 1,
1508             type => 'boolean',
1509             description => "Migration is secure using SSH tunnel by default. " .
1510               "For secure private networks you can disable it to speed up " .
1511               "migration. Deprecated, use the 'migration' property instead!",
1512         },
1513         migration => {
1514             optional => 1,
1515             type => 'string', format => $migration_format,
1516             description => "For cluster wide migration settings.",
1517         },
1518         console => {
1519             optional => 1,
1520             type => 'string',
1521             description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), an HTML5 based vnc viewer (noVNC), or an HTML5 based console client (xtermjs). If the selected viewer is not available (e.g. SPICE not activated for the VM), the fallback is noVNC.",
1522             enum => ['applet', 'vv', 'html5', 'xtermjs'],
1523         },
1524         email_from => {
1525             optional => 1,
1526             type => 'string',
1527             format => 'email-opt',
1528             description => "Specify email address to send notification from (default is root@\$hostname)",
1529         },
1530         max_workers => {
1531             optional => 1,
1532             type => 'integer',
1533             minimum => 1,
1534             description => "Defines how many workers (per node) are maximal started ".
1535               " on actions like 'stopall VMs' or task from the ha-manager.",
1536         },
1537         fencing => {
1538             optional => 1,
1539             type => 'string',
1540             default => 'watchdog',
1541             enum => [ 'watchdog', 'hardware', 'both' ],
1542             description => "Set the fencing mode of the HA cluster. Hardware mode " .
1543               "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
1544               " With both all two modes are used." .
1545               "\n\nWARNING: 'hardware' and 'both' are EXPERIMENTAL & WIP",
1546         },
1547         ha => {
1548             optional => 1,
1549             type => 'string', format => $ha_format,
1550             description => "Cluster wide HA settings.",
1551         },
1552         mac_prefix => {
1553             optional => 1,
1554             type => 'string',
1555             format => 'mac-prefix',
1556             description => 'Prefix for autogenerated MAC addresses.',
1557         },
1558         bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
1559         u2f => {
1560             optional => 1,
1561             type => 'string',
1562             format => $u2f_format,
1563             description => 'u2f',
1564         },
1565     },
1566 };
1567
1568 # make schema accessible from outside (for documentation)
1569 sub get_datacenter_schema { return $datacenter_schema };
1570
1571 sub parse_datacenter_config {
1572     my ($filename, $raw) = @_;
1573
1574     my $res = PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
1575
1576     if (my $migration = $res->{migration}) {
1577         $res->{migration} = PVE::JSONSchema::parse_property_string($migration_format, $migration);
1578     }
1579
1580     if (my $ha = $res->{ha}) {
1581         $res->{ha} = PVE::JSONSchema::parse_property_string($ha_format, $ha);
1582     }
1583
1584     # for backwards compatibility only, new migration property has precedence
1585     if (defined($res->{migration_unsecure})) {
1586         if (defined($res->{migration}->{type})) {
1587             warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
1588               "set at same time! Ignore 'migration_unsecure'\n";
1589         } else {
1590             $res->{migration}->{type} = ($res->{migration_unsecure}) ? 'insecure' : 'secure';
1591         }
1592     }
1593
1594     # for backwards compatibility only, applet maps to html5
1595     if (defined($res->{console}) && $res->{console} eq 'applet') {
1596         $res->{console} = 'html5';
1597     }
1598
1599     return $res;
1600 }
1601
1602 sub write_datacenter_config {
1603     my ($filename, $cfg) = @_;
1604
1605     # map deprecated setting to new one
1606     if (defined($cfg->{migration_unsecure}) && !defined($cfg->{migration})) {
1607         my $migration_unsecure = delete $cfg->{migration_unsecure};
1608         $cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
1609     }
1610
1611     # map deprecated applet setting to html5
1612     if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
1613         $cfg->{console} = 'html5';
1614     }
1615
1616     if (ref($cfg->{migration})) {
1617         my $migration = $cfg->{migration};
1618         $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
1619     }
1620
1621     if (ref($cfg->{ha})) {
1622         my $ha = $cfg->{ha};
1623         $cfg->{ha} = PVE::JSONSchema::print_property_string($ha, $ha_format);
1624     }
1625
1626     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1627 }
1628
1629 cfs_register_file('datacenter.cfg',
1630                   \&parse_datacenter_config,
1631                   \&write_datacenter_config);
1632
1633 # X509 Certificate cache helper
1634
1635 my $cert_cache_nodes = {};
1636 my $cert_cache_timestamp = time();
1637 my $cert_cache_fingerprints = {};
1638
1639 sub update_cert_cache {
1640     my ($update_node, $clear) = @_;
1641
1642     syslog('info', "Clearing outdated entries from certificate cache")
1643         if $clear;
1644
1645     $cert_cache_timestamp = time() if !defined($update_node);
1646
1647     my $node_list = defined($update_node) ?
1648         [ $update_node ] : [ keys %$cert_cache_nodes ];
1649
1650     foreach my $node (@$node_list) {
1651         my $clear_old = sub {
1652             if (my $old_fp = $cert_cache_nodes->{$node}) {
1653                 # distrust old fingerprint
1654                 delete $cert_cache_fingerprints->{$old_fp};
1655                 # ensure reload on next proxied request
1656                 delete $cert_cache_nodes->{$node};
1657             }
1658         };
1659
1660         my $fp = eval { get_node_fingerprint($node) };
1661         if (my $err = $@) {
1662             warn "$err\n";
1663             &$clear_old() if $clear;
1664             next;
1665         }
1666
1667         my $old_fp = $cert_cache_nodes->{$node};
1668         $cert_cache_fingerprints->{$fp} = 1;
1669         $cert_cache_nodes->{$node} = $fp;
1670
1671         if (defined($old_fp) && $fp ne $old_fp) {
1672             delete $cert_cache_fingerprints->{$old_fp};
1673         }
1674     }
1675 }
1676
1677 # load and cache cert fingerprint once
1678 sub initialize_cert_cache {
1679     my ($node) = @_;
1680
1681     update_cert_cache($node)
1682         if defined($node) && !defined($cert_cache_nodes->{$node});
1683 }
1684
1685 sub read_ssl_cert_fingerprint {
1686     my ($cert_path) = @_;
1687
1688     my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
1689         or die "unable to read '$cert_path' - $!\n";
1690
1691     my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
1692     Net::SSLeay::BIO_free($bio);
1693
1694     die "unable to read certificate from '$cert_path'\n" if !$cert;
1695
1696     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1697     Net::SSLeay::X509_free($cert);
1698
1699     die "unable to get fingerprint for '$cert_path' - got empty value\n"
1700         if !defined($fp) || $fp eq '';
1701
1702     return $fp;
1703 }
1704
1705 sub get_node_fingerprint {
1706     my ($node) = @_;
1707
1708     my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
1709     my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
1710
1711     $cert_path = $custom_cert_path if -f $custom_cert_path;
1712
1713     return read_ssl_cert_fingerprint($cert_path);
1714 }
1715
1716
1717 sub check_cert_fingerprint {
1718     my ($cert) = @_;
1719
1720     # clear cache every 30 minutes at least
1721     update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
1722
1723     # get fingerprint of server certificate
1724     my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
1725     return 0 if !defined($fp) || $fp eq ''; # error
1726
1727     my $check = sub {
1728         for my $expected (keys %$cert_cache_fingerprints) {
1729             return 1 if $fp eq $expected;
1730         }
1731         return 0;
1732     };
1733
1734     return 1 if &$check();
1735
1736     # clear cache and retry at most once every minute
1737     if (time() - $cert_cache_timestamp >= 60) {
1738         syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
1739         update_cert_cache();
1740         return &$check();
1741     }
1742
1743     return 0;
1744 }
1745
1746 # bash completion helpers
1747
1748 sub complete_next_vmid {
1749
1750     my $vmlist = get_vmlist() || {};
1751     my $idlist = $vmlist->{ids} || {};
1752
1753     for (my $i = 100; $i < 10000; $i++) {
1754         return [$i] if !defined($idlist->{$i});
1755     }
1756
1757     return [];
1758 }
1759
1760 sub complete_vmid {
1761
1762     my $vmlist = get_vmlist();
1763     my $ids = $vmlist->{ids} || {};
1764
1765     return [ keys %$ids ];
1766 }
1767
1768 sub complete_local_vmid {
1769
1770     my $vmlist = get_vmlist();
1771     my $ids = $vmlist->{ids} || {};
1772
1773     my $nodename = PVE::INotify::nodename();
1774
1775     my $res = [];
1776     foreach my $vmid (keys %$ids) {
1777         my $d = $ids->{$vmid};
1778         next if !$d->{node} || $d->{node} ne $nodename;
1779         push @$res, $vmid;
1780     }
1781
1782     return $res;
1783 }
1784
1785 sub complete_migration_target {
1786
1787     my $res = [];
1788
1789     my $nodename = PVE::INotify::nodename();
1790
1791     my $nodelist = get_nodelist();
1792     foreach my $node (@$nodelist) {
1793         next if $node eq $nodename;
1794         push @$res, $node;
1795     }
1796
1797     return $res;
1798 }
1799
1800 sub get_ssh_info {
1801     my ($node, $network_cidr) = @_;
1802
1803     my $ip;
1804     if (defined($network_cidr)) {
1805         # Use mtunnel via to get the remote node's ip inside $network_cidr.
1806         # This goes over the regular network (iow. uses get_ssh_info() with
1807         # $network_cidr undefined.
1808         # FIXME: Use the REST API client for this after creating an API entry
1809         # for get_migration_ip.
1810         my $default_remote = get_ssh_info($node, undef);
1811         my $default_ssh = ssh_info_to_command($default_remote);
1812         my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
1813             '-migration_network', $network_cidr,
1814             '-get_migration_ip'
1815         ];
1816         PVE::Tools::run_command($cmd, outfunc => sub {
1817             my ($line) = @_;
1818             chomp $line;
1819             die "internal error: unexpected output from mtunnel\n"
1820                 if defined($ip);
1821             if ($line =~ /^ip: '(.*)'$/) {
1822                 $ip = $1;
1823             } else {
1824                 die "internal error: bad output from mtunnel\n"
1825                     if defined($ip);
1826             }
1827         });
1828         die "failed to get ip for node '$node' in network '$network_cidr'\n"
1829             if !defined($ip);
1830     } else {
1831         $ip = remote_node_ip($node);
1832     }
1833
1834     return {
1835         ip => $ip,
1836         name => $node,
1837         network => $network_cidr,
1838     };
1839 }
1840
1841 sub ssh_info_to_command_base {
1842     my ($info, @extra_options) = @_;
1843     return [
1844         '/usr/bin/ssh',
1845         '-e', 'none',
1846         '-o', 'BatchMode=yes',
1847         '-o', 'HostKeyAlias='.$info->{name},
1848         @extra_options
1849     ];
1850 }
1851
1852 sub ssh_info_to_command {
1853     my ($info, @extra_options) = @_;
1854     my $cmd = ssh_info_to_command_base($info, @extra_options);
1855     push @$cmd, "root\@$info->{ip}";
1856     return $cmd;
1857 }
1858
1859 my $corosync_link_format = {
1860     address => {
1861         default_key => 1,
1862         type => 'string', format => 'address',
1863         format_description => 'IP',
1864         description => "Hostname (or IP) of this corosync link address.",
1865     },
1866     priority => {
1867         optional => 1,
1868         type => 'integer',
1869         minimum => 0,
1870         maximum => 255,
1871         default => 0,
1872         description => "The priority for the link when knet is used in 'passive' mode. Lower value means higher priority.",
1873     },
1874 };
1875 my $corosync_link_desc = {
1876     type => 'string', format => $corosync_link_format,
1877     description => "Address and priority information of a single corosync link.",
1878     optional => 1,
1879 };
1880 PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);
1881
1882 sub parse_corosync_link {
1883     my ($value) = @_;
1884
1885     return undef if !defined($value);
1886
1887     return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
1888 }
1889
1890 sub assert_joinable {
1891     my ($local_addr, $link0, $link1, $force) = @_;
1892
1893     my $errors = '';
1894     my $error = sub { $errors .= "* $_[0]\n"; };
1895
1896     if (-f $authfile) {
1897         $error->("authentication key '$authfile' already exists");
1898     }
1899
1900     if (-f $clusterconf)  {
1901         $error->("cluster config '$clusterconf' already exists");
1902     }
1903
1904     my $vmlist = get_vmlist();
1905     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
1906         $error->("this host already contains virtual guests");
1907     }
1908
1909     if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
1910         $error->("corosync is already running, is this node already in a cluster?!");
1911     }
1912
1913     # check if corosync ring IPs are configured on the current nodes interfaces
1914     my $check_ip = sub {
1915         my $ip = shift // return;
1916         my $logid = shift;
1917         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
1918             my $host = $ip;
1919             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
1920             if ($@) {
1921                 $error->("$logid: cannot use '$host': $@\n") ;
1922                 return;
1923             }
1924         }
1925
1926         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
1927         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
1928
1929         $error->("$logid: cannot use IP '$ip', it must be configured exactly once on local node!\n")
1930             if (scalar(@$configured_ips) != 1);
1931     };
1932
1933     $check_ip->($local_addr, 'local node address');
1934     $check_ip->($link0->{address}, 'ring0') if defined($link0);
1935     $check_ip->($link1->{address}, 'ring1') if defined($link1);
1936
1937     if ($errors) {
1938         warn "detected the following error(s):\n$errors";
1939         die "Check if node may join a cluster failed!\n" if !$force;
1940     }
1941 }
1942
1943 # NOTE: filesystem must be offline here, no DB changes allowed
1944 my $backup_cfs_database = sub {
1945     my ($dbfile) = @_;
1946
1947     mkdir $dbbackupdir;
1948
1949     my $ctime = time();
1950     my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
1951
1952     print "backup old database to '$backup_fn'\n";
1953
1954     my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \ ">${backup_fn}"] ];
1955     run_command($cmd, 'errmsg' => "cannot backup old database\n");
1956
1957     my $maxfiles = 10; # purge older backup
1958     my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql.gz> ];
1959
1960     if ((my $count = scalar(@$backups)) > $maxfiles) {
1961         foreach my $f (@$backups[$maxfiles..$count-1]) {
1962             next if $f !~ m/^(\S+)$/; # untaint
1963             print "delete old backup '$1'\n";
1964             unlink $1;
1965         }
1966     }
1967 };
1968
1969 sub join {
1970     my ($param) = @_;
1971
1972     my $nodename = PVE::INotify::nodename();
1973     my $local_ip_address = remote_node_ip($nodename);
1974
1975     my $link0 = parse_corosync_link($param->{link0});
1976     my $link1 = parse_corosync_link($param->{link1});
1977
1978     # check if we can join with the given parameters and current node state
1979     assert_joinable($local_ip_address, $link0, $link1, $param->{force});
1980
1981     setup_sshd_config();
1982     setup_rootsshconfig();
1983     setup_ssh_keys();
1984
1985     # make sure known_hosts is on local filesystem
1986     ssh_unmerge_known_hosts();
1987
1988     my $host = $param->{hostname};
1989     my $conn_args = {
1990         username => 'root@pam',
1991         password => $param->{password},
1992         cookie_name => 'PVEAuthCookie',
1993         protocol => 'https',
1994         host => $host,
1995         port => 8006,
1996     };
1997
1998     if (my $fp = $param->{fingerprint}) {
1999         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
2000     } else {
2001         # API schema ensures that we can only get here from CLI handler
2002         $conn_args->{manual_verification} = 1;
2003     }
2004
2005     print "Establishing API connection with host '$host'\n";
2006
2007     my $conn = PVE::APIClient::LWP->new(%$conn_args);
2008     $conn->login();
2009
2010     # login raises an exception on failure, so if we get here we're good
2011     print "Login succeeded.\n";
2012
2013     my $args = {};
2014     $args->{force} = $param->{force} if defined($param->{force});
2015     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
2016     $args->{votes} = $param->{votes} if defined($param->{votes});
2017     # just pass the un-parsed string through, or as we've address as the
2018     # default_key, we can just pass the fallback directly too
2019     $args->{link0} = $param->{link0} // $local_ip_address;
2020     $args->{link1} = $param->{link1} if defined($param->{link1});
2021
2022     print "Request addition of this node\n";
2023     my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
2024
2025     print "Join request OK, finishing setup locally\n";
2026
2027     # added successfuly - now prepare local node
2028     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
2029 }
2030
2031 sub finish_join {
2032     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
2033
2034     mkdir "$localclusterdir";
2035     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
2036     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
2037
2038     print "stopping pve-cluster service\n";
2039     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
2040     run_command($cmd, errmsg => "can't stop pve-cluster service");
2041
2042     $backup_cfs_database->($dbfile);
2043     unlink $dbfile;
2044
2045     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
2046     run_command($cmd, errmsg => "starting pve-cluster failed");
2047
2048     # wait for quorum
2049     my $printqmsg = 1;
2050     while (!check_cfs_quorum(1)) {
2051         if ($printqmsg) {
2052             print "waiting for quorum...";
2053             STDOUT->flush();
2054             $printqmsg = 0;
2055         }
2056         sleep(1);
2057     }
2058     print "OK\n" if !$printqmsg;
2059
2060     updatecerts_and_ssh(1);
2061
2062     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
2063     run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
2064
2065     print "successfully added node '$nodename' to cluster.\n";
2066 }
2067
2068 sub updatecerts_and_ssh {
2069     my ($force_new_cert, $silent) = @_;
2070
2071     my $p = sub { print "$_[0]\n" if !$silent };
2072
2073     setup_rootsshconfig();
2074
2075     gen_pve_vzdump_symlink();
2076
2077     if (!check_cfs_quorum(1)) {
2078         return undef if $silent;
2079         die "no quorum - unable to update files\n";
2080     }
2081
2082     setup_ssh_keys();
2083
2084     my $nodename = PVE::INotify::nodename();
2085     my $local_ip_address = remote_node_ip($nodename);
2086
2087     $p->("(re)generate node files");
2088     $p->("generate new node certificate") if $force_new_cert;
2089     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
2090
2091     $p->("merge authorized SSH keys and known hosts");
2092     ssh_merge_keys();
2093     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
2094     gen_pve_vzdump_files();
2095 }
2096
2097 1;