data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Encode;
   7 use File::stat qw();
   8 use File::Path qw(make_path);
   9 use JSON;
  10 use Net::SSLeay;
  11 use POSIX qw(ENOENT);
  12 use Socket;
  13 use Storable qw(dclone);
  14
  15 use PVE::Certificate;
  16 use PVE::INotify;
  17 use PVE::IPCC;
  18 use PVE::JSONSchema;
  19 use PVE::Network;
  20 use PVE::SafeSyslog;
  21 use PVE::Tools qw(run_command);
  22
  23 use PVE::Cluster::IPCConst;
  24
  25 use base 'Exporter';
  26
  27 our @EXPORT_OK = qw(
  28 cfs_read_file
  29 cfs_write_file
  30 cfs_register_file
  31 cfs_lock_file);
  32
  33 # x509 certificate utils
  34
  35 my $basedir = "/etc/pve";
  36 my $authdir = "$basedir/priv";
  37 my $lockdir = "/etc/pve/priv/lock";
  38
  39 # cfs and corosync files
  40 my $dbfile = "/var/lib/pve-cluster/config.db";
  41 my $dbbackupdir = "/var/lib/pve-cluster/backup";
  42
  43 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
  44 # observed files are the one we can get directly through IPCC, they are cached
  45 # using a computed version and only those can be used by the cfs_*_file methods
  46 my $observed = {
  47     'vzdump.cron' => 1,
  48     'storage.cfg' => 1,
  49     'datacenter.cfg' => 1,
  50     'replication.cfg' => 1,
  51     'corosync.conf' => 1,
  52     'corosync.conf.new' => 1,
  53     'user.cfg' => 1,
  54     'domains.cfg' => 1,
  55     'priv/shadow.cfg' => 1,
  56     'priv/tfa.cfg' => 1,
  57     'priv/token.cfg' => 1,
  58     'priv/acme/plugins.cfg' => 1,
  59     'priv/ipam.db' => 1,
  60     '/qemu-server/' => 1,
  61     '/openvz/' => 1,
  62     '/lxc/' => 1,
  63     'ha/crm_commands' => 1,
  64     'ha/manager_status' => 1,
  65     'ha/resources.cfg' => 1,
  66     'ha/groups.cfg' => 1,
  67     'ha/fence.cfg' => 1,
  68     'status.cfg' => 1,
  69     'ceph.conf' => 1,
  70     'sdn/vnets.cfg' => 1,
  71     'sdn/zones.cfg' => 1,
  72     'sdn/controllers.cfg' => 1,
  73     'sdn/subnets.cfg' => 1,
  74     'sdn/ipams.cfg' => 1,
  75     'sdn/dns.cfg' => 1,
  76     'sdn/.running-config' => 1,
  77     'virtual-guest/cpu-models.conf' => 1,
  78 };
  79
  80 sub prepare_observed_file_basedirs {
  81
  82     if (!check_cfs_is_mounted(1)) {
  83         warn "pmxcfs isn't mounted (/etc/pve), chickening out..\n";
  84         return;
  85     }
  86
  87     for my $f (sort keys %$observed) {
  88         next if $f !~ m!^(.*)/[^/]+$!;
  89         my $dir = "$basedir/$1";
  90         next if -e $dir; # can also be a link, so just use -e xist check
  91         print "creating directory '$dir' for observerd files\n";
  92         make_path($dir);
  93     }
  94 }
  95
  96 sub base_dir {
  97     return $basedir;
  98 }
  99
 100 sub auth_dir {
 101     return $authdir;
 102 }
 103
 104 sub check_cfs_quorum {
 105     my ($noerr) = @_;
 106
 107     # note: -w filename always return 1 for root, so wee need
 108     # to use File::lstat here
 109     my $st = File::stat::lstat("$basedir/local");
 110     my $quorate = ($st && (($st->mode & 0200) != 0));
 111
 112     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 113
 114     return $quorate;
 115 }
 116
 117 sub check_cfs_is_mounted {
 118     my ($noerr) = @_;
 119
 120     my $res = -l "$basedir/local";
 121
 122     die "pve configuration filesystem not mounted\n"
 123         if !$res && !$noerr;
 124
 125     return $res;
 126 }
 127
 128 my $versions = {};
 129 my $vmlist = {};
 130 my $clinfo = {};
 131
 132 my $ipcc_send_rec = sub {
 133     my ($msgid, $data) = @_;
 134
 135     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 136
 137     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 138
 139     return $res;
 140 };
 141
 142 my $ipcc_send_rec_json = sub {
 143     my ($msgid, $data) = @_;
 144
 145     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 146
 147     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 148
 149     return decode_json($res);
 150 };
 151
 152 my $ipcc_get_config = sub {
 153     my ($path) = @_;
 154
 155     my $bindata = pack "Z*", $path;
 156     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_CONFIG, $bindata);
 157     if (!defined($res)) {
 158         if ($! != 0) {
 159             return undef if $! == ENOENT;
 160             die "$!\n";
 161         }
 162         return '';
 163     }
 164
 165     return $res;
 166 };
 167
 168 my $ipcc_get_status = sub {
 169     my ($name, $nodename) = @_;
 170
 171     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 172     return PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_STATUS, $bindata);
 173 };
 174
 175 my $ipcc_remove_status = sub {
 176     my ($name) = @_;
 177     # we just omit the data payload, pmxcfs takes this as hint and removes this
 178     # key from the status hashtable
 179     my $bindata = pack "Z[256]", $name;
 180     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 181 };
 182
 183 my $ipcc_update_status = sub {
 184     my ($name, $data) = @_;
 185
 186     my $raw = ref($data) ? encode_json($data) : $data;
 187     # update status
 188     my $bindata = pack "Z[256]Z*", $name, $raw;
 189
 190     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 191 };
 192
 193 my $ipcc_log = sub {
 194     my ($priority, $ident, $tag, $msg) = @_;
 195
 196     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 197     bytes::length($tag) + 1, $ident, $tag, $msg;
 198
 199     return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG, $bindata);
 200 };
 201
 202 my $ipcc_get_cluster_log = sub {
 203     my ($user, $max) = @_;
 204
 205     $max = 0 if !defined($max);
 206
 207     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 208     return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG, $bindata);
 209 };
 210
 211 my $ipcc_verify_token = sub {
 212     my ($full_token) = @_;
 213
 214     my $bindata = pack "Z*", $full_token;
 215     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_VERIFY_TOKEN, $bindata);
 216
 217     return 1 if $! == 0;
 218     return 0 if $! == ENOENT;
 219
 220     die "$!\n";
 221 };
 222
 223 my $ccache = {};
 224
 225 sub cfs_update {
 226     my ($fail) = @_;
 227     eval {
 228         my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION);
 229         die "no starttime\n" if !$res->{starttime};
 230
 231         if (!$res->{starttime} || !$versions->{starttime} ||
 232             $res->{starttime} != $versions->{starttime}) {
 233             #print "detected changed starttime\n";
 234             $vmlist = {};
 235             $clinfo = {};
 236             $ccache = {};
 237         }
 238
 239         $versions = $res;
 240     };
 241     my $err = $@;
 242     if ($err) {
 243         $versions = {};
 244         $vmlist = {};
 245         $clinfo = {};
 246         $ccache = {};
 247         die $err if $fail;
 248         warn $err;
 249     }
 250
 251     eval {
 252         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 253             #warn "detected new clinfo\n";
 254             $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO);
 255         }
 256     };
 257     $err = $@;
 258     if ($err) {
 259         $clinfo = {};
 260         die $err if $fail;
 261         warn $err;
 262     }
 263
 264     eval {
 265         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 266             #warn "detected new vmlist1\n";
 267             $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST);
 268         }
 269     };
 270     $err = $@;
 271     if ($err) {
 272         $vmlist = {};
 273         die $err if $fail;
 274         warn $err;
 275     }
 276 }
 277
 278 sub get_vmlist {
 279     return $vmlist;
 280 }
 281
 282 sub get_clinfo {
 283     return $clinfo;
 284 }
 285
 286 sub get_members {
 287     return $clinfo->{nodelist};
 288 }
 289
 290 sub get_nodelist {
 291     my $nodelist = $clinfo->{nodelist};
 292
 293     my $nodename = PVE::INotify::nodename();
 294
 295     if (!$nodelist || !$nodelist->{$nodename}) {
 296         return [ $nodename ];
 297     }
 298
 299     return [ keys %$nodelist ];
 300 }
 301
 302 # only stored in a in-memory hashtable inside pmxcfs, local data is gone after
 303 # a restart (of pmxcfs or the node), peer data is still available then
 304 # best used for status data, like running (ceph) services, package versions, ...
 305 sub broadcast_node_kv {
 306     my ($key, $data) = @_;
 307
 308     if (!defined($data)) {
 309         eval {
 310             $ipcc_remove_status->("kv/$key");
 311         };
 312     } else {
 313         die "cannot send a reference\n" if ref($data);
 314         my $size = length($data);
 315         die "data for '$key' too big\n" if $size >= (32 * 1024); # limit from pmxfs
 316
 317         eval {
 318             $ipcc_update_status->("kv/$key", $data);
 319         };
 320     }
 321
 322     warn $@ if $@;
 323 }
 324
 325 # nodename is optional
 326 sub get_node_kv {
 327     my ($key, $nodename) = @_;
 328
 329     my $res = {};
 330     my $get_node_data = sub {
 331         my ($node) = @_;
 332         my $raw = $ipcc_get_status->("kv/$key", $node);
 333         $res->{$node} = unpack("Z*", $raw) if $raw;
 334     };
 335
 336     if ($nodename) {
 337         $get_node_data->($nodename);
 338     } else {
 339         my $nodelist = get_nodelist();
 340
 341         foreach my $node (@$nodelist) {
 342             $get_node_data->($node);
 343         }
 344     }
 345
 346     return $res;
 347 }
 348
 349 # property: a config property you want to get, e.g., this is perfect to get
 350 # the 'lock' entry of a guest _fast_ (>100 faster than manual parsing here)
 351 # vmid: optipnal, if a valid is passed we only check that one, else return all
 352 # NOTE: does *not* searches snapshot and PENDING entries sections!
 353 sub get_guest_config_property {
 354     my ($property, $vmid) = @_;
 355
 356     die "property is required" if !defined($property);
 357
 358     my $bindata = pack "VZ*", $vmid // 0, $property;
 359     my $res = $ipcc_send_rec_json->(CFS_IPC_GET_GUEST_CONFIG_PROPERTY, $bindata);
 360
 361     return $res;
 362 }
 363
 364 # $data must be a chronological descending ordered array of tasks
 365 sub broadcast_tasklist {
 366     my ($data) = @_;
 367
 368     # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
 369     # from pmxcfs) - drop older items until we satisfy this constraint
 370     my $size = length(encode_json($data));
 371     while ($size >= (32 * 1024)) {
 372         pop @$data;
 373         $size = length(encode_json($data));
 374     }
 375
 376     eval {
 377         &$ipcc_update_status("tasklist", $data);
 378     };
 379
 380     warn $@ if $@;
 381 }
 382
 383 my $tasklistcache = {};
 384
 385 sub get_tasklist {
 386     my ($nodename) = @_;
 387
 388     my $kvstore = $versions->{kvstore} || {};
 389
 390     my $nodelist = get_nodelist();
 391
 392     my $res = [];
 393     foreach my $node (@$nodelist) {
 394         next if $nodename && ($nodename ne $node);
 395         eval {
 396             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 397             my $cache = $tasklistcache->{$node};
 398             if (!$cache || !$ver || !$cache->{version} || ($cache->{version} != $ver)) {
 399                 my $tasks = [];
 400                 if (my $raw = $ipcc_get_status->("tasklist", $node)) {
 401                     my $json_str = unpack("Z*", $raw);
 402                     $tasks = decode_json($json_str);
 403                 }
 404                 push @$res, @$tasks;
 405                 $tasklistcache->{$node} = {
 406                     data => $tasks,
 407                     version => $ver,
 408                 };
 409             } elsif ($cache && $cache->{data}) {
 410                 push @$res, $cache->{data}->@*;
 411             }
 412         };
 413         my $err = $@;
 414         syslog('err', $err) if $err;
 415     }
 416
 417     return $res;
 418 }
 419
 420 sub broadcast_rrd {
 421     my ($rrdid, $data) = @_;
 422
 423     eval {
 424         &$ipcc_update_status("rrd/$rrdid", $data);
 425     };
 426     my $err = $@;
 427
 428     warn $err if $err;
 429 }
 430
 431 my $last_rrd_dump = 0;
 432 my $last_rrd_data = "";
 433
 434 sub rrd_dump {
 435
 436     my $ctime = time();
 437
 438     my $diff = $ctime - $last_rrd_dump;
 439     if ($diff < 2) {
 440         return $last_rrd_data;
 441     }
 442
 443     my $raw;
 444     eval {
 445         $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP);
 446     };
 447     my $err = $@;
 448
 449     if ($err) {
 450         warn $err;
 451         return {};
 452     }
 453
 454     my $res = {};
 455
 456     if ($raw) {
 457         while ($raw =~ s/^(.*)\n//) {
 458             my ($key, @ela) = split(/:/, $1);
 459             next if !$key;
 460             next if !(scalar(@ela) > 1);
 461             $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
 462         }
 463     }
 464
 465     $last_rrd_dump = $ctime;
 466     $last_rrd_data = $res;
 467
 468     return $res;
 469 }
 470
 471
 472 # a fast way to read files (avoid fuse overhead)
 473 sub get_config {
 474     my ($path) = @_;
 475
 476     return &$ipcc_get_config($path);
 477 }
 478
 479 sub get_cluster_log {
 480     my ($user, $max) = @_;
 481
 482     return &$ipcc_get_cluster_log($user, $max);
 483 }
 484
 485 sub verify_token {
 486     my ($userid, $token) = @_;
 487
 488     return &$ipcc_verify_token("$userid $token");
 489 }
 490
 491 my $file_info = {};
 492
 493 sub cfs_register_file {
 494     my ($filename, $parser, $writer) = @_;
 495
 496     $observed->{$filename} || die "unknown file '$filename'";
 497
 498     die "file '$filename' already registered" if $file_info->{$filename};
 499
 500     $file_info->{$filename} = {
 501         parser => $parser,
 502         writer => $writer,
 503     };
 504 }
 505
 506 my $ccache_read = sub {
 507     my ($filename, $parser, $version) = @_;
 508
 509     $ccache->{$filename} = {} if !$ccache->{$filename};
 510
 511     my $ci = $ccache->{$filename};
 512
 513     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 514         # we always call the parser, even when the file does not exist
 515         # (in that case $data is undef)
 516         my $data = get_config($filename);
 517         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 518         $ci->{version} = $version;
 519     }
 520
 521     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 522
 523     return $res;
 524 };
 525
 526 sub cfs_file_version {
 527     my ($filename) = @_;
 528
 529     my $version;
 530     my $infotag;
 531     if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
 532         my ($type, $vmid) = ($1, $2);
 533         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 534             $version = $vmlist->{ids}->{$vmid}->{version};
 535         }
 536         $infotag = "/$type/";
 537     } else {
 538         $infotag = $filename;
 539         $version = $versions->{$filename};
 540     }
 541
 542     my $info = $file_info->{$infotag} ||
 543         die "unknown file type '$filename'\n";
 544
 545     return wantarray ? ($version, $info) : $version;
 546 }
 547
 548 sub cfs_read_file {
 549     my ($filename) = @_;
 550
 551     my ($version, $info) = cfs_file_version($filename);
 552     my $parser = $info->{parser};
 553
 554     return &$ccache_read($filename, $parser, $version);
 555 }
 556
 557 sub cfs_write_file {
 558     my ($filename, $data) = @_;
 559
 560     my ($version, $info) = cfs_file_version($filename);
 561
 562     my $writer = $info->{writer} || die "no writer defined";
 563
 564     my $fsname = "/etc/pve/$filename";
 565
 566     my $raw = &$writer($fsname, $data);
 567
 568     if (my $ci = $ccache->{$filename}) {
 569         $ci->{version} = undef;
 570     }
 571
 572     PVE::Tools::file_set_contents($fsname, $raw);
 573 }
 574
 575 my $cfs_lock = sub {
 576     my ($lockid, $timeout, $code, @param) = @_;
 577
 578     my $prev_alarm = alarm(0); # suspend outer alarm early
 579
 580     my $res;
 581     my $got_lock = 0;
 582
 583     # this timeout is for acquire the lock
 584     $timeout = 10 if !$timeout;
 585
 586     my $filename = "$lockdir/$lockid";
 587
 588     my $is_code_err = 0;
 589     eval {
 590
 591         mkdir $lockdir;
 592
 593         if (! -d $lockdir) {
 594             die "pve cluster filesystem not online.\n";
 595         }
 596
 597         my $timeout_err = sub { die "got lock request timeout\n"; };
 598         local $SIG{ALRM} = $timeout_err;
 599
 600         while (1) {
 601             alarm ($timeout);
 602             $got_lock = mkdir($filename);
 603             $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
 604
 605             last if $got_lock;
 606
 607             $timeout_err->() if $timeout <= 0;
 608
 609             print STDERR "trying to acquire cfs lock '$lockid' ...\n";
 610             utime (0, 0, $filename); # cfs unlock request
 611             sleep(1);
 612         }
 613
 614         # fixed command timeout: cfs locks have a timeout of 120
 615         # using 60 gives us another 60 seconds to abort the task
 616         local $SIG{ALRM} = sub { die "'$lockid'-locked command timed out - aborting\n"; };
 617         alarm(60);
 618
 619         cfs_update(); # make sure we read latest versions inside code()
 620
 621         $is_code_err = 1; # allows to differ between locking and actual-work errors
 622
 623         $res = &$code(@param);
 624
 625         alarm(0);
 626     };
 627
 628     my $err = $@;
 629
 630     $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
 631
 632     rmdir $filename if $got_lock; # if we held the lock always unlock again
 633
 634     alarm($prev_alarm);
 635
 636     if ($err) {
 637         if (ref($err) eq 'PVE::Exception' || $is_code_err) {
 638             # re-raise defined exceptions
 639             $@ = $err;
 640         } else {
 641             # add lock info for plain errors comming from the locking itself
 642             $@ = "cfs-lock '$lockid' error: $err";
 643         }
 644         return undef;
 645     }
 646
 647     $@ = undef;
 648
 649     return $res;
 650 };
 651
 652 sub cfs_lock_file {
 653     my ($filename, $timeout, $code, @param) = @_;
 654
 655     my $info = $observed->{$filename} || die "unknown file '$filename'";
 656
 657     my $lockid = "file-$filename";
 658     $lockid =~ s/[.\/]/_/g;
 659
 660     &$cfs_lock($lockid, $timeout, $code, @param);
 661 }
 662
 663 sub cfs_lock_storage {
 664     my ($storeid, $timeout, $code, @param) = @_;
 665
 666     my $lockid = "storage-$storeid";
 667
 668     &$cfs_lock($lockid, $timeout, $code, @param);
 669 }
 670
 671 sub cfs_lock_domain {
 672     my ($domainname, $timeout, $code, @param) = @_;
 673
 674     my $lockid = "domain-$domainname";
 675
 676     &$cfs_lock($lockid, $timeout, $code, @param);
 677 }
 678
 679 sub cfs_lock_acme {
 680     my ($account, $timeout, $code, @param) = @_;
 681
 682     my $lockid = "acme-$account";
 683
 684     &$cfs_lock($lockid, $timeout, $code, @param);
 685 }
 686
 687 sub cfs_lock_authkey {
 688     my ($timeout, $code, @param) = @_;
 689
 690     $cfs_lock->('authkey', $timeout, $code, @param);
 691 }
 692
 693 sub cfs_lock_firewall {
 694     my ($scope, $timeout, $code, @param) = @_;
 695
 696     my $lockid = "firewall-$scope";
 697
 698     $cfs_lock->($lockid, $timeout, $code, @param);
 699 }
 700
 701 my $log_levels = {
 702     "emerg" => 0,
 703     "alert" => 1,
 704     "crit" => 2,
 705     "critical" => 2,
 706     "err" => 3,
 707     "error" => 3,
 708     "warn" => 4,
 709     "warning" => 4,
 710     "notice" => 5,
 711     "info" => 6,
 712     "debug" => 7,
 713 };
 714
 715 sub log_msg {
 716    my ($priority, $ident, $msg) = @_;
 717
 718    if (my $tmp = $log_levels->{$priority}) {
 719        $priority = $tmp;
 720    }
 721
 722    die "need numeric log priority" if $priority !~ /^\d+$/;
 723
 724    my $tag = PVE::SafeSyslog::tag();
 725
 726    $msg = "empty message" if !$msg;
 727
 728    $ident = "" if !$ident;
 729    $ident = encode("ascii", $ident,
 730                    sub { sprintf "\\u%04x", shift });
 731
 732    my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
 733
 734    if ($ident) {
 735        syslog($priority, "<%s> %s", $ident, $ascii);
 736    } else {
 737        syslog($priority, "%s", $ascii);
 738    }
 739
 740    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
 741
 742    syslog("err", "writing cluster log failed: $@") if $@;
 743 }
 744
 745 sub check_vmid_unused {
 746     my ($vmid, $noerr) = @_;
 747
 748     my $vmlist = get_vmlist();
 749
 750     my $d = $vmlist->{ids}->{$vmid};
 751     return 1 if !defined($d);
 752
 753     return undef if $noerr;
 754
 755     my $vmtypestr =  $d->{type} eq 'qemu' ? 'VM' : 'CT';
 756     die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
 757 }
 758
 759 sub check_node_exists {
 760     my ($nodename, $noerr) = @_;
 761
 762     my $nodelist = $clinfo->{nodelist};
 763     return 1 if $nodelist && $nodelist->{$nodename};
 764
 765     return undef if $noerr;
 766
 767     die "no such cluster node '$nodename'\n";
 768 }
 769
 770 # this is also used to get the IP of the local node
 771 sub remote_node_ip {
 772     my ($nodename, $noerr) = @_;
 773
 774     my $nodelist = $clinfo->{nodelist};
 775     if ($nodelist && $nodelist->{$nodename}) {
 776         if (my $ip = $nodelist->{$nodename}->{ip}) {
 777             return $ip if !wantarray;
 778             my $family = $nodelist->{$nodename}->{address_family};
 779             if (!$family) {
 780                 $nodelist->{$nodename}->{address_family} =
 781                     $family =
 782                     PVE::Tools::get_host_address_family($ip);
 783             }
 784             return wantarray ? ($ip, $family) : $ip;
 785         }
 786     }
 787
 788     # fallback: try to get IP by other means
 789     return PVE::Network::get_ip_from_hostname($nodename, $noerr);
 790 }
 791
 792 sub get_node_fingerprint {
 793     my ($node) = @_;
 794
 795     my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
 796     my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
 797
 798     $cert_path = $custom_cert_path if -f $custom_cert_path;
 799
 800     return PVE::Certificate::get_certificate_fingerprint($cert_path);
 801 }
 802
 803 # bash completion helpers
 804
 805 sub complete_next_vmid {
 806
 807     my $vmlist = get_vmlist() || {};
 808     my $idlist = $vmlist->{ids} || {};
 809
 810     for (my $i = 100; $i < 10000; $i++) {
 811         return [$i] if !defined($idlist->{$i});
 812     }
 813
 814     return [];
 815 }
 816
 817 sub complete_vmid {
 818
 819     my $vmlist = get_vmlist();
 820     my $ids = $vmlist->{ids} || {};
 821
 822     return [ keys %$ids ];
 823 }
 824
 825 sub complete_local_vmid {
 826
 827     my $vmlist = get_vmlist();
 828     my $ids = $vmlist->{ids} || {};
 829
 830     my $nodename = PVE::INotify::nodename();
 831
 832     my $res = [];
 833     foreach my $vmid (keys %$ids) {
 834         my $d = $ids->{$vmid};
 835         next if !$d->{node} || $d->{node} ne $nodename;
 836         push @$res, $vmid;
 837     }
 838
 839     return $res;
 840 }
 841
 842 sub complete_migration_target {
 843
 844     my $res = [];
 845
 846     my $nodename = PVE::INotify::nodename();
 847
 848     my $nodelist = get_nodelist();
 849     foreach my $node (@$nodelist) {
 850         next if $node eq $nodename;
 851         push @$res, $node;
 852     }
 853
 854     return $res;
 855 }
 856
 857
 858 # NOTE: filesystem must be offline here, no DB changes allowed
 859 sub cfs_backup_database {
 860     mkdir $dbbackupdir;
 861
 862     my $ctime = time();
 863     my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
 864
 865     print "backup old database to '$backup_fn'\n";
 866
 867     my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \ ">${backup_fn}"] ];
 868     run_command($cmd, 'errmsg' => "cannot backup old database\n");
 869
 870     my $maxfiles = 10; # purge older backup
 871     my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql.gz> ];
 872
 873     if ((my $count = scalar(@$backups)) > $maxfiles) {
 874         foreach my $f (@$backups[$maxfiles..$count-1]) {
 875             next if $f !~ m/^(\S+)$/; # untaint
 876             print "delete old backup '$1'\n";
 877             unlink $1;
 878         }
 879     }
 880
 881     return $dbfile;
 882 }
 883
 884 1;