data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Encode;
   7 use File::stat qw();
   8 use File::Path qw(make_path);
   9 use JSON;
  10 use Net::SSLeay;
  11 use POSIX qw(ENOENT);
  12 use Socket;
  13 use Storable qw(dclone);
  14
  15 use PVE::Certificate;
  16 use PVE::INotify;
  17 use PVE::IPCC;
  18 use PVE::JSONSchema;
  19 use PVE::Network;
  20 use PVE::SafeSyslog;
  21 use PVE::Tools qw(run_command);
  22
  23 use PVE::Cluster::IPCConst;
  24
  25 use base 'Exporter';
  26
  27 our @EXPORT_OK = qw(
  28 cfs_read_file
  29 cfs_write_file
  30 cfs_register_file
  31 cfs_lock_file);
  32
  33 # x509 certificate utils
  34
  35 my $basedir = "/etc/pve";
  36 my $authdir = "$basedir/priv";
  37 my $lockdir = "/etc/pve/priv/lock";
  38
  39 # cfs and corosync files
  40 my $dbfile = "/var/lib/pve-cluster/config.db";
  41 my $dbbackupdir = "/var/lib/pve-cluster/backup";
  42
  43 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
  44 # observed files are the one we can get directly through IPCC, they are cached
  45 # using a computed version and only those can be used by the cfs_*_file methods
  46 my $observed = {
  47     'vzdump.cron' => 1,
  48     'jobs.cfg' => 1,
  49     'storage.cfg' => 1,
  50     'datacenter.cfg' => 1,
  51     'replication.cfg' => 1,
  52     'corosync.conf' => 1,
  53     'corosync.conf.new' => 1,
  54     'firewall/cluster.fw' => 1,
  55     'user.cfg' => 1,
  56     'domains.cfg' => 1,
  57     'priv/shadow.cfg' => 1,
  58     'priv/tfa.cfg' => 1,
  59     'priv/token.cfg' => 1,
  60     'priv/acme/plugins.cfg' => 1,
  61     'priv/ipam.db' => 1,
  62     '/qemu-server/' => 1,
  63     '/openvz/' => 1,
  64     '/lxc/' => 1,
  65     'ha/crm_commands' => 1,
  66     'ha/manager_status' => 1,
  67     'ha/resources.cfg' => 1,
  68     'ha/groups.cfg' => 1,
  69     'ha/fence.cfg' => 1,
  70     'status.cfg' => 1,
  71     'ceph.conf' => 1,
  72     'sdn/vnets.cfg' => 1,
  73     'sdn/zones.cfg' => 1,
  74     'sdn/controllers.cfg' => 1,
  75     'sdn/subnets.cfg' => 1,
  76     'sdn/ipams.cfg' => 1,
  77     'sdn/dns.cfg' => 1,
  78     'sdn/.running-config' => 1,
  79     'virtual-guest/cpu-models.conf' => 1,
  80 };
  81
  82 sub prepare_observed_file_basedirs {
  83
  84     if (!check_cfs_is_mounted(1)) {
  85         warn "pmxcfs isn't mounted (/etc/pve), chickening out..\n";
  86         return;
  87     }
  88
  89     for my $f (sort keys %$observed) {
  90         next if $f !~ m!^(.*)/[^/]+$!;
  91         my $dir = "$basedir/$1";
  92         next if -e $dir; # can also be a link, so just use -e xist check
  93         print "creating directory '$dir' for observerd files\n";
  94         make_path($dir);
  95     }
  96 }
  97
  98 sub base_dir {
  99     return $basedir;
 100 }
 101
 102 sub auth_dir {
 103     return $authdir;
 104 }
 105
 106 sub check_cfs_quorum {
 107     my ($noerr) = @_;
 108
 109     # note: -w filename always return 1 for root, so wee need
 110     # to use File::lstat here
 111     my $st = File::stat::lstat("$basedir/local");
 112     my $quorate = ($st && (($st->mode & 0200) != 0));
 113
 114     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 115
 116     return $quorate;
 117 }
 118
 119 sub check_cfs_is_mounted {
 120     my ($noerr) = @_;
 121
 122     my $res = -l "$basedir/local";
 123
 124     die "pve configuration filesystem (pmxcfs) not mounted\n" if !$res && !$noerr;
 125
 126     return $res;
 127 }
 128
 129 my $versions = {};
 130 my $vmlist = {};
 131 my $clinfo = {};
 132
 133 my $ipcc_send_rec = sub {
 134     my ($msgid, $data) = @_;
 135
 136     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 137
 138     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 139
 140     return $res;
 141 };
 142
 143 my $ipcc_send_rec_json = sub {
 144     my ($msgid, $data) = @_;
 145
 146     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 147
 148     die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
 149
 150     return decode_json($res);
 151 };
 152
 153 my $ipcc_get_config = sub {
 154     my ($path) = @_;
 155
 156     my $bindata = pack "Z*", $path;
 157     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_CONFIG, $bindata);
 158     if (!defined($res)) {
 159         if ($! != 0) {
 160             return undef if $! == ENOENT;
 161             die "$!\n";
 162         }
 163         return '';
 164     }
 165
 166     return $res;
 167 };
 168
 169 my $ipcc_get_status = sub {
 170     my ($name, $nodename) = @_;
 171
 172     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 173     return PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_STATUS, $bindata);
 174 };
 175
 176 my $ipcc_remove_status = sub {
 177     my ($name) = @_;
 178     # we just omit the data payload, pmxcfs takes this as hint and removes this
 179     # key from the status hashtable
 180     my $bindata = pack "Z[256]", $name;
 181     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 182 };
 183
 184 my $ipcc_update_status = sub {
 185     my ($name, $data) = @_;
 186
 187     my $raw = ref($data) ? encode_json($data) : $data;
 188     # update status
 189     my $bindata = pack "Z[256]Z*", $name, $raw;
 190
 191     return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
 192 };
 193
 194 my $ipcc_log = sub {
 195     my ($priority, $ident, $tag, $msg) = @_;
 196
 197     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 198     bytes::length($tag) + 1, $ident, $tag, $msg;
 199
 200     return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG, $bindata);
 201 };
 202
 203 my $ipcc_get_cluster_log = sub {
 204     my ($user, $max) = @_;
 205
 206     $max = 0 if !defined($max);
 207
 208     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 209     return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG, $bindata);
 210 };
 211
 212 my $ipcc_verify_token = sub {
 213     my ($full_token) = @_;
 214
 215     my $bindata = pack "Z*", $full_token;
 216     my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_VERIFY_TOKEN, $bindata);
 217
 218     return 1 if $! == 0;
 219     return 0 if $! == ENOENT;
 220
 221     die "$!\n";
 222 };
 223
 224 my $ccache = {};
 225
 226 sub cfs_update {
 227     my ($fail) = @_;
 228     eval {
 229         my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION);
 230         die "no starttime\n" if !$res->{starttime};
 231
 232         if (!$res->{starttime} || !$versions->{starttime} ||
 233             $res->{starttime} != $versions->{starttime}) {
 234             #print "detected changed starttime\n";
 235             $vmlist = {};
 236             $clinfo = {};
 237             $ccache = {};
 238         }
 239
 240         $versions = $res;
 241     };
 242     my $err = $@;
 243     if ($err) {
 244         $versions = {};
 245         $vmlist = {};
 246         $clinfo = {};
 247         $ccache = {};
 248         die $err if $fail;
 249         warn $err;
 250     }
 251
 252     eval {
 253         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 254             #warn "detected new clinfo\n";
 255             $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO);
 256         }
 257     };
 258     $err = $@;
 259     if ($err) {
 260         $clinfo = {};
 261         die $err if $fail;
 262         warn $err;
 263     }
 264
 265     eval {
 266         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 267             #warn "detected new vmlist1\n";
 268             $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST);
 269         }
 270     };
 271     $err = $@;
 272     if ($err) {
 273         $vmlist = {};
 274         die $err if $fail;
 275         warn $err;
 276     }
 277 }
 278
 279 sub get_vmlist {
 280     return $vmlist;
 281 }
 282
 283 sub get_clinfo {
 284     return $clinfo;
 285 }
 286
 287 sub get_members {
 288     return $clinfo->{nodelist};
 289 }
 290
 291 sub get_nodelist {
 292     my $nodelist = $clinfo->{nodelist};
 293
 294     my $nodename = PVE::INotify::nodename();
 295
 296     if (!$nodelist || !$nodelist->{$nodename}) {
 297         return [ $nodename ];
 298     }
 299
 300     return [ keys %$nodelist ];
 301 }
 302
 303 # only stored in a in-memory hashtable inside pmxcfs, local data is gone after
 304 # a restart (of pmxcfs or the node), peer data is still available then
 305 # best used for status data, like running (ceph) services, package versions, ...
 306 sub broadcast_node_kv {
 307     my ($key, $data) = @_;
 308
 309     if (!defined($data)) {
 310         eval { $ipcc_remove_status->("kv/$key") };
 311     } else {
 312         die "cannot send a reference\n" if ref($data);
 313         my $size = length($data);
 314         die "data for '$key' too big\n" if $size >= (32 * 1024); # limit from pmxfs
 315
 316         eval { $ipcc_update_status->("kv/$key", $data) };
 317     }
 318     warn $@ if $@;
 319 }
 320
 321 # nodename is optional
 322 sub get_node_kv {
 323     my ($key, $nodename) = @_;
 324
 325     my $res = {};
 326     my $get_node_data = sub {
 327         my ($node) = @_;
 328         my $raw = $ipcc_get_status->("kv/$key", $node);
 329         $res->{$node} = unpack("Z*", $raw) if $raw;
 330     };
 331
 332     if ($nodename) {
 333         $get_node_data->($nodename);
 334     } else {
 335         for my $node (get_nodelist()->@*) {
 336             $get_node_data->($node);
 337         }
 338     }
 339
 340     return $res;
 341 }
 342
 343 # properties: an array-ref of config properties you want to get, e.g., this
 344 # is perfect to get multiple properties of a guest _fast_
 345 # (>100 faster than manual parsing here)
 346 # vmid: optional, if a valid is passed we only check that one, else return all
 347 # NOTE: does *not* searches snapshot and PENDING entries sections!
 348 # NOTE: returns the guest config lines (excluding trailing whitespace) as is,
 349 #       so for non-trivial properties, checking the validity must be done
 350 # NOTE: no permission check is done, that is the responsibilty of the caller
 351 sub get_guest_config_properties {
 352     my ($properties, $vmid) = @_;
 353
 354     die "properties required" if !defined($properties);
 355
 356     my $num_props = scalar(@$properties);
 357     die "only up to 255 properties supported" if $num_props > 255;
 358     my $bindata = pack "VC", $vmid // 0, $num_props;
 359     for my $property (@$properties) {
 360         $bindata .= pack "Z*", $property;
 361     }
 362     my $res = $ipcc_send_rec_json->(CFS_IPC_GET_GUEST_CONFIG_PROPERTIES, $bindata);
 363
 364     return $res;
 365 }
 366
 367 # property: a config property you want to get, e.g., this is perfect to get
 368 # the 'lock' entry of a guest _fast_ (>100 faster than manual parsing here)
 369 # vmid: optional, if a valid is passed we only check that one, else return all
 370 # NOTE: does *not* searches snapshot and PENDING entries sections!
 371 # NOTE: returns the guest config lines (excluding trailing whitespace) as is,
 372 #       so for non-trivial properties, checking the validity must be done
 373 # NOTE: no permission check is done, that is the responsibilty of the caller
 374 sub get_guest_config_property {
 375     my ($property, $vmid) = @_;
 376
 377     die "property is required" if !defined($property);
 378
 379     my $bindata = pack "VZ*", $vmid // 0, $property;
 380     my $res = $ipcc_send_rec_json->(CFS_IPC_GET_GUEST_CONFIG_PROPERTY, $bindata);
 381
 382     return $res;
 383 }
 384
 385 # $data must be a chronological descending ordered array of tasks
 386 sub broadcast_tasklist {
 387     my ($data) = @_;
 388
 389     # the serialized list may not get bigger than 128 KiB (CFS_MAX_STATUS_SIZE from pmxcfs)
 390     # drop older items until we satisfy this constraint
 391     my $size = length(encode_json($data));
 392     while ($size >= (32 * 1024)) { # TODO: update to 128 KiB in PVE 8.x
 393         pop @$data;
 394         $size = length(encode_json($data));
 395     }
 396
 397     eval { $ipcc_update_status->("tasklist", $data) };
 398     warn $@ if $@;
 399 }
 400
 401 my $tasklistcache = {};
 402
 403 sub get_tasklist {
 404     my ($nodename) = @_;
 405
 406     my $kvstore = $versions->{kvstore} || {};
 407
 408     my $nodelist = get_nodelist();
 409
 410     my $res = [];
 411     foreach my $node (@$nodelist) {
 412         next if $nodename && ($nodename ne $node);
 413         eval {
 414             my $ver = exists $kvstore->{$node} ? $kvstore->{$node}->{tasklist} : undef;
 415             my $cache = $tasklistcache->{$node};
 416             if (!$cache || !$ver || !$cache->{version} || ($cache->{version} != $ver)) {
 417                 my $tasks = [];
 418                 if (my $raw = $ipcc_get_status->("tasklist", $node)) {
 419                     my $json_str = unpack("Z*", $raw);
 420                     $tasks = decode_json($json_str);
 421                 }
 422                 push @$res, @$tasks;
 423                 $tasklistcache->{$node} = {
 424                     data => $tasks,
 425                     version => $ver,
 426                 };
 427             } elsif ($cache && $cache->{data}) {
 428                 push @$res, $cache->{data}->@*;
 429             }
 430         };
 431         my $err = $@;
 432         syslog('err', $err) if $err;
 433     }
 434
 435     return $res;
 436 }
 437
 438 sub broadcast_rrd {
 439     my ($rrdid, $data) = @_;
 440
 441     eval {
 442         &$ipcc_update_status("rrd/$rrdid", $data);
 443     };
 444     my $err = $@;
 445
 446     warn $err if $err;
 447 }
 448
 449 my $last_rrd_dump = 0;
 450 my $last_rrd_data = "";
 451
 452 sub rrd_dump {
 453
 454     my $ctime = time();
 455
 456     my $diff = $ctime - $last_rrd_dump;
 457     if ($diff < 2) {
 458         return $last_rrd_data;
 459     }
 460
 461     my $raw;
 462     eval {
 463         $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP);
 464     };
 465     my $err = $@;
 466
 467     if ($err) {
 468         warn $err;
 469         return {};
 470     }
 471
 472     my $res = {};
 473
 474     if ($raw) {
 475         while ($raw =~ s/^(.*)\n//) {
 476             my ($key, @ela) = split(/:/, $1);
 477             next if !$key;
 478             next if !(scalar(@ela) > 1);
 479             $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
 480         }
 481     }
 482
 483     $last_rrd_dump = $ctime;
 484     $last_rrd_data = $res;
 485
 486     return $res;
 487 }
 488
 489
 490 # a fast way to read files (avoid fuse overhead)
 491 sub get_config {
 492     my ($path) = @_;
 493
 494     return &$ipcc_get_config($path);
 495 }
 496
 497 sub get_cluster_log {
 498     my ($user, $max) = @_;
 499
 500     return &$ipcc_get_cluster_log($user, $max);
 501 }
 502
 503 sub verify_token {
 504     my ($userid, $token) = @_;
 505
 506     return &$ipcc_verify_token("$userid $token");
 507 }
 508
 509 my $file_info = {};
 510
 511 sub cfs_register_file {
 512     my ($filename, $parser, $writer) = @_;
 513
 514     $observed->{$filename} || die "unknown file '$filename'";
 515
 516     die "file '$filename' already registered" if $file_info->{$filename};
 517
 518     $file_info->{$filename} = {
 519         parser => $parser,
 520         writer => $writer,
 521     };
 522 }
 523
 524 my $ccache_read = sub {
 525     my ($filename, $parser, $version) = @_;
 526
 527     $ccache->{$filename} = {} if !$ccache->{$filename};
 528
 529     my $ci = $ccache->{$filename};
 530
 531     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 532         # we always call the parser, even when the file does not exist
 533         # (in that case $data is undef)
 534         my $data = get_config($filename);
 535         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 536         $ci->{version} = $version;
 537     }
 538
 539     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 540
 541     return $res;
 542 };
 543
 544 sub cfs_file_version {
 545     my ($filename) = @_;
 546
 547     my $version;
 548     my $infotag;
 549     if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
 550         my ($type, $vmid) = ($1, $2);
 551         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 552             $version = $vmlist->{ids}->{$vmid}->{version};
 553         }
 554         $infotag = "/$type/";
 555     } else {
 556         $infotag = $filename;
 557         $version = $versions->{$filename};
 558     }
 559
 560     my $info = $file_info->{$infotag} ||
 561         die "unknown file type '$filename'\n";
 562
 563     return wantarray ? ($version, $info) : $version;
 564 }
 565
 566 sub cfs_read_file {
 567     my ($filename) = @_;
 568
 569     my ($version, $info) = cfs_file_version($filename);
 570     my $parser = $info->{parser};
 571
 572     return &$ccache_read($filename, $parser, $version);
 573 }
 574
 575 sub cfs_write_file {
 576     my ($filename, $data) = @_;
 577
 578     my ($version, $info) = cfs_file_version($filename);
 579
 580     my $writer = $info->{writer} || die "no writer defined";
 581
 582     my $fsname = "/etc/pve/$filename";
 583
 584     my $raw = &$writer($fsname, $data);
 585
 586     if (my $ci = $ccache->{$filename}) {
 587         $ci->{version} = undef;
 588     }
 589
 590     PVE::Tools::file_set_contents($fsname, $raw);
 591 }
 592
 593 my $cfs_lock = sub {
 594     my ($lockid, $timeout, $code, @param) = @_;
 595
 596     my $prev_alarm = alarm(0); # suspend outer alarm early
 597
 598     my $res;
 599     my $got_lock = 0;
 600
 601     # this timeout is for acquire the lock
 602     $timeout = 10 if !$timeout;
 603
 604     my $filename = "$lockdir/$lockid";
 605
 606     my $is_code_err = 0;
 607     eval {
 608
 609         mkdir $lockdir;
 610
 611         if (! -d $lockdir) {
 612             die "pve cluster filesystem not online.\n";
 613         }
 614
 615         my $timeout_err = sub { die "got lock request timeout\n"; };
 616         local $SIG{ALRM} = $timeout_err;
 617
 618         while (1) {
 619             alarm ($timeout);
 620             $got_lock = mkdir($filename);
 621             $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
 622
 623             last if $got_lock;
 624
 625             $timeout_err->() if $timeout <= 0;
 626
 627             print STDERR "trying to acquire cfs lock '$lockid' ...\n";
 628             utime (0, 0, $filename); # cfs unlock request
 629             sleep(1);
 630         }
 631
 632         # fixed command timeout: cfs locks have a timeout of 120
 633         # using 60 gives us another 60 seconds to abort the task
 634         local $SIG{ALRM} = sub { die "'$lockid'-locked command timed out - aborting\n"; };
 635         alarm(60);
 636
 637         cfs_update(); # make sure we read latest versions inside code()
 638
 639         $is_code_err = 1; # allows to differ between locking and actual-work errors
 640
 641         $res = &$code(@param);
 642
 643         alarm(0);
 644     };
 645
 646     my $err = $@;
 647
 648     $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
 649
 650     rmdir $filename if $got_lock; # if we held the lock always unlock again
 651
 652     alarm($prev_alarm);
 653
 654     if ($err) {
 655         if (ref($err) eq 'PVE::Exception' || $is_code_err) {
 656             # re-raise defined exceptions
 657             $@ = $err;
 658         } else {
 659             # add lock info for plain errors comming from the locking itself
 660             $@ = "cfs-lock '$lockid' error: $err";
 661         }
 662         return undef;
 663     }
 664
 665     $@ = undef;
 666
 667     return $res;
 668 };
 669
 670 sub cfs_lock_file {
 671     my ($filename, $timeout, $code, @param) = @_;
 672
 673     my $info = $observed->{$filename} || die "unknown file '$filename'";
 674
 675     my $lockid = "file-$filename";
 676     $lockid =~ s/[.\/]/_/g;
 677
 678     &$cfs_lock($lockid, $timeout, $code, @param);
 679 }
 680
 681 sub cfs_lock_storage {
 682     my ($storeid, $timeout, $code, @param) = @_;
 683
 684     my $lockid = "storage-$storeid";
 685
 686     &$cfs_lock($lockid, $timeout, $code, @param);
 687 }
 688
 689 sub cfs_lock_domain {
 690     my ($domainname, $timeout, $code, @param) = @_;
 691
 692     my $lockid = "domain-$domainname";
 693
 694     &$cfs_lock($lockid, $timeout, $code, @param);
 695 }
 696
 697 sub cfs_lock_acme {
 698     my ($account, $timeout, $code, @param) = @_;
 699
 700     my $lockid = "acme-$account";
 701
 702     &$cfs_lock($lockid, $timeout, $code, @param);
 703 }
 704
 705 sub cfs_lock_authkey {
 706     my ($timeout, $code, @param) = @_;
 707
 708     $cfs_lock->('authkey', $timeout, $code, @param);
 709 }
 710
 711 sub cfs_lock_firewall {
 712     my ($scope, $timeout, $code, @param) = @_;
 713
 714     my $lockid = "firewall-$scope";
 715
 716     $cfs_lock->($lockid, $timeout, $code, @param);
 717 }
 718
 719 my $log_levels = {
 720     "emerg" => 0,
 721     "alert" => 1,
 722     "crit" => 2,
 723     "critical" => 2,
 724     "err" => 3,
 725     "error" => 3,
 726     "warn" => 4,
 727     "warning" => 4,
 728     "notice" => 5,
 729     "info" => 6,
 730     "debug" => 7,
 731 };
 732
 733 sub log_msg {
 734    my ($priority, $ident, $msg) = @_;
 735
 736    if (my $tmp = $log_levels->{$priority}) {
 737        $priority = $tmp;
 738    }
 739
 740    die "need numeric log priority" if $priority !~ /^\d+$/;
 741
 742    my $tag = PVE::SafeSyslog::tag();
 743
 744    $msg = "empty message" if !$msg;
 745
 746    $ident = "" if !$ident;
 747    $ident = encode("ascii", $ident,
 748                    sub { sprintf "\\u%04x", shift });
 749
 750    my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
 751
 752    if ($ident) {
 753        syslog($priority, "<%s> %s", $ident, $ascii);
 754    } else {
 755        syslog($priority, "%s", $ascii);
 756    }
 757
 758    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
 759
 760    syslog("err", "writing cluster log failed: $@") if $@;
 761 }
 762
 763 sub check_vmid_unused {
 764     my ($vmid, $noerr) = @_;
 765
 766     my $vmlist = get_vmlist();
 767
 768     my $d = $vmlist->{ids}->{$vmid};
 769     return 1 if !defined($d);
 770
 771     return undef if $noerr;
 772
 773     my $vmtypestr =  $d->{type} eq 'qemu' ? 'VM' : 'CT';
 774     die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
 775 }
 776
 777 sub check_node_exists {
 778     my ($nodename, $noerr) = @_;
 779
 780     my $nodelist = $clinfo->{nodelist};
 781     return 1 if $nodelist && $nodelist->{$nodename};
 782
 783     return undef if $noerr;
 784
 785     die "no such cluster node '$nodename'\n";
 786 }
 787
 788 # this is also used to get the IP of the local node
 789 sub remote_node_ip {
 790     my ($nodename, $noerr) = @_;
 791
 792     my $nodelist = $clinfo->{nodelist};
 793     if ($nodelist && $nodelist->{$nodename}) {
 794         if (my $ip = $nodelist->{$nodename}->{ip}) {
 795             return $ip if !wantarray;
 796             my $family = $nodelist->{$nodename}->{address_family};
 797             if (!$family) {
 798                 $nodelist->{$nodename}->{address_family} =
 799                     $family =
 800                     PVE::Tools::get_host_address_family($ip);
 801             }
 802             return wantarray ? ($ip, $family) : $ip;
 803         }
 804     }
 805
 806     # fallback: try to get IP by other means
 807     return PVE::Network::get_ip_from_hostname($nodename, $noerr);
 808 }
 809
 810 sub get_node_fingerprint {
 811     my ($node) = @_;
 812
 813     my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
 814     my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
 815
 816     $cert_path = $custom_cert_path if -f $custom_cert_path;
 817
 818     return PVE::Certificate::get_certificate_fingerprint($cert_path);
 819 }
 820
 821 # bash completion helpers
 822
 823 sub complete_next_vmid {
 824
 825     my $vmlist = get_vmlist() || {};
 826     my $idlist = $vmlist->{ids} || {};
 827
 828     for (my $i = 100; $i < 10000; $i++) {
 829         return [$i] if !defined($idlist->{$i});
 830     }
 831
 832     return [];
 833 }
 834
 835 sub complete_vmid {
 836
 837     my $vmlist = get_vmlist();
 838     my $ids = $vmlist->{ids} || {};
 839
 840     return [ keys %$ids ];
 841 }
 842
 843 sub complete_local_vmid {
 844
 845     my $vmlist = get_vmlist();
 846     my $ids = $vmlist->{ids} || {};
 847
 848     my $nodename = PVE::INotify::nodename();
 849
 850     my $res = [];
 851     foreach my $vmid (keys %$ids) {
 852         my $d = $ids->{$vmid};
 853         next if !$d->{node} || $d->{node} ne $nodename;
 854         push @$res, $vmid;
 855     }
 856
 857     return $res;
 858 }
 859
 860 sub complete_migration_target {
 861
 862     my $res = [];
 863
 864     my $nodename = PVE::INotify::nodename();
 865
 866     my $nodelist = get_nodelist();
 867     foreach my $node (@$nodelist) {
 868         next if $node eq $nodename;
 869         push @$res, $node;
 870     }
 871
 872     return $res;
 873 }
 874
 875
 876 # NOTE: filesystem must be offline here, no DB changes allowed
 877 sub cfs_backup_database {
 878     mkdir $dbbackupdir;
 879
 880     my $ctime = time();
 881     my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
 882
 883     print "backup old database to '$backup_fn'\n";
 884
 885     my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \ ">${backup_fn}"] ];
 886     run_command($cmd, 'errmsg' => "cannot backup old database\n");
 887
 888     my $maxfiles = 10; # purge older backup
 889     my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql.gz> ];
 890
 891     if ((my $count = scalar(@$backups)) > $maxfiles) {
 892         foreach my $f (@$backups[$maxfiles..$count-1]) {
 893             next if $f !~ m/^(\S+)$/; # untaint
 894             print "delete old backup '$1'\n";
 895             unlink $1;
 896         }
 897     }
 898
 899     return $dbfile;
 900 }
 901
 902 1;