data/PVE/Cluster.pm

   1 package PVE::Cluster;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EEXIST);
   6 use File::stat qw();
   7 use Socket;
   8 use Storable qw(dclone);
   9 use IO::File;
  10 use MIME::Base64;
  11 use XML::Parser;
  12 use Digest::SHA;
  13 use Digest::HMAC_SHA1;
  14 use PVE::Tools;
  15 use PVE::INotify;
  16 use PVE::IPCC;
  17 use PVE::SafeSyslog;
  18 use PVE::JSONSchema;
  19 use JSON;
  20 use RRDs;
  21 use Encode;
  22 use base 'Exporter';
  23
  24 our @EXPORT_OK = qw(
  25 cfs_read_file
  26 cfs_write_file
  27 cfs_register_file
  28 cfs_lock_file);
  29
  30 use Data::Dumper; # fixme: remove
  31
  32 # x509 certificate utils
  33
  34 my $basedir = "/etc/pve";
  35 my $authdir = "$basedir/priv";
  36 my $lockdir = "/etc/pve/priv/lock";
  37
  38 my $authprivkeyfn = "$authdir/authkey.key";
  39 my $authpubkeyfn = "$basedir/authkey.pub";
  40 my $pveca_key_fn = "$authdir/pve-root-ca.key";
  41 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
  42 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
  43 # this is just a secret accessable by the web browser
  44 # and is used for CSRF prevention
  45 my $pvewww_key_fn = "$basedir/pve-www.key";
  46
  47 # ssh related files
  48 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  49 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  50 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  51 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  52 my $sshknownhosts = "/etc/pve/priv/known_hosts";
  53 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
  54 my $sshd_config_fn = "/etc/ssh/sshd_config";
  55 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  56 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  57 my $rootsshconfig = "/root/.ssh/config";
  58
  59 my $observed = {
  60     'vzdump.cron' => 1,
  61     'storage.cfg' => 1,
  62     'datacenter.cfg' => 1,
  63     'corosync.conf' => 1,
  64     'corosync.conf.new' => 1,
  65     'user.cfg' => 1,
  66     'domains.cfg' => 1,
  67     'priv/shadow.cfg' => 1,
  68     '/qemu-server/' => 1,
  69     '/openvz/' => 1,
  70     '/lxc/' => 1,
  71     'ha/crm_commands' => 1,
  72     'ha/manager_status' => 1,
  73     'ha/resources.cfg' => 1,
  74     'ha/groups.cfg' => 1,
  75 };
  76
  77 # only write output if something fails
  78 sub run_silent_cmd {
  79     my ($cmd) = @_;
  80
  81     my $outbuf = '';
  82
  83     my $record_output = sub {
  84         $outbuf .= shift;
  85         $outbuf .= "\n";
  86     };
  87
  88     eval {
  89         PVE::Tools::run_command($cmd, outfunc => $record_output,
  90                                 errfunc => $record_output);
  91     };
  92
  93     my $err = $@;
  94
  95     if ($err) {
  96         print STDERR $outbuf;
  97         die $err;
  98     }
  99 }
 100
 101 sub check_cfs_quorum {
 102     my ($noerr) = @_;
 103
 104     # note: -w filename always return 1 for root, so wee need
 105     # to use File::lstat here
 106     my $st = File::stat::lstat("$basedir/local");
 107     my $quorate = ($st && (($st->mode & 0200) != 0));
 108
 109     die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
 110
 111     return $quorate;
 112 }
 113
 114 sub check_cfs_is_mounted {
 115     my ($noerr) = @_;
 116
 117     my $res = -l "$basedir/local";
 118
 119     die "pve configuration filesystem not mounted\n"
 120         if !$res && !$noerr;
 121
 122     return $res;
 123 }
 124
 125 sub gen_local_dirs {
 126     my ($nodename) = @_;
 127
 128     check_cfs_is_mounted();
 129
 130     my @required_dirs = (
 131         "$basedir/priv",
 132         "$basedir/nodes",
 133         "$basedir/nodes/$nodename",
 134         "$basedir/nodes/$nodename/lxc",
 135         "$basedir/nodes/$nodename/qemu-server",
 136         "$basedir/nodes/$nodename/openvz",
 137         "$basedir/nodes/$nodename/priv");
 138
 139     foreach my $dir (@required_dirs) {
 140         if (! -d $dir) {
 141             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 142         }
 143     }
 144 }
 145
 146 sub gen_auth_key {
 147
 148     return if -f "$authprivkeyfn";
 149
 150     check_cfs_is_mounted();
 151
 152     mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
 153
 154     my $cmd = "openssl genrsa -out '$authprivkeyfn' 2048";
 155     run_silent_cmd($cmd);
 156
 157     $cmd = "openssl rsa -in '$authprivkeyfn' -pubout -out '$authpubkeyfn'";
 158     run_silent_cmd($cmd)
 159 }
 160
 161 sub gen_pveca_key {
 162
 163     return if -f $pveca_key_fn;
 164
 165     eval {
 166         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '2048']);
 167     };
 168
 169     die "unable to generate pve ca key:\n$@" if $@;
 170 }
 171
 172 sub gen_pveca_cert {
 173
 174     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 175         return 0;
 176     }
 177
 178     gen_pveca_key();
 179
 180     # we try to generate an unique 'subject' to avoid browser problems
 181     # (reused serial numbers, ..)
 182     my $nid = (split (/\s/, `md5sum '$pveca_key_fn'`))[0] || time();
 183
 184     eval {
 185         run_silent_cmd(['openssl', 'req', '-batch', '-days', '3650', '-new',
 186                         '-x509', '-nodes', '-key',
 187                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 188                         "/CN=Proxmox Virtual Environment/OU=$nid/O=PVE Cluster Manager CA/"]);
 189     };
 190
 191     die "generating pve root certificate failed:\n$@" if $@;
 192
 193     return 1;
 194 }
 195
 196 sub gen_pve_ssl_key {
 197     my ($nodename) = @_;
 198
 199     die "no node name specified" if !$nodename;
 200
 201     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 202
 203     return if -f $pvessl_key_fn;
 204
 205     eval {
 206         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 207     };
 208
 209     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 210 }
 211
 212 sub gen_pve_www_key {
 213
 214     return if -f $pvewww_key_fn;
 215
 216     eval {
 217         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 218     };
 219
 220     die "unable to generate pve www key:\n$@" if $@;
 221 }
 222
 223 sub update_serial {
 224     my ($serial) = @_;
 225
 226     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 227 }
 228
 229 sub gen_pve_ssl_cert {
 230     my ($force, $nodename, $ip) = @_;
 231
 232     die "no node name specified" if !$nodename;
 233     die "no IP specified" if !$ip;
 234
 235     my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
 236
 237     return if !$force && -f $pvessl_cert_fn;
 238
 239     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 240
 241     my $rc = PVE::INotify::read_file('resolvconf');
 242
 243     $names .= ",IP:$ip";
 244
 245     my $fqdn = $nodename;
 246
 247     $names .= ",DNS:$nodename";
 248
 249     if ($rc && $rc->{search}) {
 250         $fqdn = $nodename . "." . $rc->{search};
 251         $names .= ",DNS:$fqdn";
 252     }
 253
 254     my $sslconf = <<__EOD;
 255 RANDFILE = /root/.rnd
 256 extensions = v3_req
 257
 258 [ req ]
 259 default_bits = 2048
 260 distinguished_name = req_distinguished_name
 261 req_extensions = v3_req
 262 prompt = no
 263 string_mask = nombstr
 264
 265 [ req_distinguished_name ]
 266 organizationalUnitName = PVE Cluster Node
 267 organizationName = Proxmox Virtual Environment
 268 commonName = $fqdn
 269
 270 [ v3_req ]
 271 basicConstraints = CA:FALSE
 272 nsCertType = server
 273 keyUsage = nonRepudiation, digitalSignature, keyEncipherment
 274 subjectAltName = $names
 275 __EOD
 276
 277     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 278     my $fh = IO::File->new ($cfgfn, "w");
 279     print $fh $sslconf;
 280     close ($fh);
 281
 282     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 283     unlink $reqfn;
 284
 285     my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
 286     eval {
 287         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 288                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 289     };
 290
 291     if (my $err = $@) {
 292         unlink $reqfn;
 293         unlink $cfgfn;
 294         die "unable to generate pve certificate request:\n$err";
 295     }
 296
 297     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 298
 299     eval {
 300         run_silent_cmd(['openssl', 'x509', '-req', '-in', $reqfn, '-days', '3650',
 301                         '-out', $pvessl_cert_fn, '-CAkey', $pveca_key_fn,
 302                         '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn,
 303                         '-extfile', $cfgfn]);
 304     };
 305
 306     if (my $err = $@) {
 307         unlink $reqfn;
 308         unlink $cfgfn;
 309         die "unable to generate pve ssl certificate:\n$err";
 310     }
 311
 312     unlink $cfgfn;
 313     unlink $reqfn;
 314 }
 315
 316 sub gen_pve_node_files {
 317     my ($nodename, $ip, $opt_force) = @_;
 318
 319     gen_local_dirs($nodename);
 320
 321     gen_auth_key();
 322
 323     # make sure we have a (cluster wide) secret
 324     # for CSRFR prevention
 325     gen_pve_www_key();
 326
 327     # make sure we have a (per node) private key
 328     gen_pve_ssl_key($nodename);
 329
 330     # make sure we have a CA
 331     my $force = gen_pveca_cert();
 332
 333     $force = 1 if $opt_force;
 334
 335     gen_pve_ssl_cert($force, $nodename, $ip);
 336 }
 337
 338 my $vzdump_cron_dummy = <<__EOD;
 339 # cluster wide vzdump cron schedule
 340 # Atomatically generated file - do not edit
 341
 342 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 343
 344 __EOD
 345
 346 sub gen_pve_vzdump_symlink {
 347
 348     my $filename = "/etc/pve/vzdump.cron";
 349
 350     my $link_fn = "/etc/cron.d/vzdump";
 351
 352     if ((-f $filename) && (! -l $link_fn)) {
 353         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 354         symlink($filename, $link_fn);
 355     }
 356 }
 357
 358 sub gen_pve_vzdump_files {
 359
 360     my $filename = "/etc/pve/vzdump.cron";
 361
 362     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 363         if ! -f $filename;
 364
 365     gen_pve_vzdump_symlink();
 366 };
 367
 368 my $versions = {};
 369 my $vmlist = {};
 370 my $clinfo = {};
 371
 372 my $ipcc_send_rec = sub {
 373     my ($msgid, $data) = @_;
 374
 375     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 376
 377     die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
 378
 379     return $res;
 380 };
 381
 382 my $ipcc_send_rec_json = sub {
 383     my ($msgid, $data) = @_;
 384
 385     my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
 386
 387     die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
 388
 389     return decode_json($res);
 390 };
 391
 392 my $ipcc_get_config = sub {
 393     my ($path) = @_;
 394
 395     my $bindata = pack "Z*", $path;
 396     my $res = PVE::IPCC::ipcc_send_rec(6, $bindata);
 397     if (!defined($res)) {
 398         return undef if ($! != 0);
 399         return '';
 400     }
 401
 402     return $res;
 403 };
 404
 405 my $ipcc_get_status = sub {
 406     my ($name, $nodename) = @_;
 407
 408     my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
 409     return PVE::IPCC::ipcc_send_rec(5, $bindata);
 410 };
 411
 412 my $ipcc_update_status = sub {
 413     my ($name, $data) = @_;
 414
 415     my $raw = ref($data) ? encode_json($data) : $data;
 416     # update status
 417     my $bindata = pack "Z[256]Z*", $name, $raw;
 418
 419     return &$ipcc_send_rec(4, $bindata);
 420 };
 421
 422 my $ipcc_log = sub {
 423     my ($priority, $ident, $tag, $msg) = @_;
 424
 425     my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
 426     bytes::length($tag) + 1, $ident, $tag, $msg;
 427
 428     return &$ipcc_send_rec(7, $bindata);
 429 };
 430
 431 my $ipcc_get_cluster_log = sub {
 432     my ($user, $max) = @_;
 433
 434     $max = 0 if !defined($max);
 435
 436     my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
 437     return &$ipcc_send_rec(8, $bindata);
 438 };
 439
 440 my $ccache = {};
 441
 442 sub cfs_update {
 443     eval {
 444         my $res = &$ipcc_send_rec_json(1);
 445         #warn "GOT1: " . Dumper($res);
 446         die "no starttime\n" if !$res->{starttime};
 447
 448         if (!$res->{starttime} || !$versions->{starttime} ||
 449             $res->{starttime} != $versions->{starttime}) {
 450             #print "detected changed starttime\n";
 451             $vmlist = {};
 452             $clinfo = {};
 453             $ccache = {};
 454         }
 455
 456         $versions = $res;
 457     };
 458     my $err = $@;
 459     if ($err) {
 460         $versions = {};
 461         $vmlist = {};
 462         $clinfo = {};
 463         $ccache = {};
 464         warn $err;
 465     }
 466
 467     eval {
 468         if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
 469             #warn "detected new clinfo\n";
 470             $clinfo = &$ipcc_send_rec_json(2);
 471         }
 472     };
 473     $err = $@;
 474     if ($err) {
 475         $clinfo = {};
 476         warn $err;
 477     }
 478
 479     eval {
 480         if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
 481             #warn "detected new vmlist1\n";
 482             $vmlist = &$ipcc_send_rec_json(3);
 483         }
 484     };
 485     $err = $@;
 486     if ($err) {
 487         $vmlist = {};
 488         warn $err;
 489     }
 490 }
 491
 492 sub get_vmlist {
 493     return $vmlist;
 494 }
 495
 496 sub get_clinfo {
 497     return $clinfo;
 498 }
 499
 500 sub get_members {
 501     return $clinfo->{nodelist};
 502 }
 503
 504 sub get_nodelist {
 505
 506     my $nodelist = $clinfo->{nodelist};
 507
 508     my $result = [];
 509
 510     my $nodename = PVE::INotify::nodename();
 511
 512     if (!$nodelist || !$nodelist->{$nodename}) {
 513         return [ $nodename ];
 514     }
 515
 516     return [ keys %$nodelist ];
 517 }
 518
 519 sub broadcast_tasklist {
 520     my ($data) = @_;
 521
 522     eval {
 523         &$ipcc_update_status("tasklist", $data);
 524     };
 525
 526     warn $@ if $@;
 527 }
 528
 529 my $tasklistcache = {};
 530
 531 sub get_tasklist {
 532     my ($nodename) = @_;
 533
 534     my $kvstore = $versions->{kvstore} || {};
 535
 536     my $nodelist = get_nodelist();
 537
 538     my $res = [];
 539     foreach my $node (@$nodelist) {
 540         next if $nodename && ($nodename ne $node);
 541         eval {
 542             my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
 543             my $cd = $tasklistcache->{$node};
 544             if (!$cd || !$ver || !$cd->{version} ||
 545                 ($cd->{version} != $ver)) {
 546                 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
 547                 my $data = decode_json($raw);
 548                 push @$res, @$data;
 549                 $cd = $tasklistcache->{$node} = {
 550                     data => $data,
 551                     version => $ver,
 552                 };
 553             } elsif ($cd && $cd->{data}) {
 554                 push @$res, @{$cd->{data}};
 555             }
 556         };
 557         my $err = $@;
 558         syslog('err', $err) if $err;
 559     }
 560
 561     return $res;
 562 }
 563
 564 sub broadcast_rrd {
 565     my ($rrdid, $data) = @_;
 566
 567     eval {
 568         &$ipcc_update_status("rrd/$rrdid", $data);
 569     };
 570     my $err = $@;
 571
 572     warn $err if $err;
 573 }
 574
 575 my $last_rrd_dump = 0;
 576 my $last_rrd_data = "";
 577
 578 sub rrd_dump {
 579
 580     my $ctime = time();
 581
 582     my $diff = $ctime - $last_rrd_dump;
 583     if ($diff < 2) {
 584         return $last_rrd_data;
 585     }
 586
 587     my $raw;
 588     eval {
 589         $raw = &$ipcc_send_rec(10);
 590     };
 591     my $err = $@;
 592
 593     if ($err) {
 594         warn $err;
 595         return {};
 596     }
 597
 598     my $res = {};
 599
 600     if ($raw) {
 601         while ($raw =~ s/^(.*)\n//) {
 602             my ($key, @ela) = split(/:/, $1);
 603             next if !$key;
 604             next if !(scalar(@ela) > 1);
 605             $res->{$key} = \@ela;
 606         }
 607     }
 608
 609     $last_rrd_dump = $ctime;
 610     $last_rrd_data = $res;
 611
 612     return $res;
 613 }
 614
 615 sub create_rrd_data {
 616     my ($rrdname, $timeframe, $cf) = @_;
 617
 618     my $rrddir = "/var/lib/rrdcached/db";
 619
 620     my $rrd = "$rrddir/$rrdname";
 621
 622     my $setup = {
 623         hour =>  [ 60, 70 ],
 624         day  =>  [ 60*30, 70 ],
 625         week =>  [ 60*180, 70 ],
 626         month => [ 60*720, 70 ],
 627         year =>  [ 60*10080, 70 ],
 628     };
 629
 630     my ($reso, $count) = @{$setup->{$timeframe}};
 631     my $ctime  = $reso*int(time()/$reso);
 632     my $req_start = $ctime - $reso*$count;
 633
 634     $cf = "AVERAGE" if !$cf;
 635
 636     my @args = (
 637         "-s" => $req_start,
 638         "-e" => $ctime - 1,
 639         "-r" => $reso,
 640         );
 641
 642     my $socket = "/var/run/rrdcached.sock";
 643     push @args, "--daemon" => "unix:$socket" if -S $socket;
 644
 645     my ($start, $step, $names, $data) = RRDs::fetch($rrd, $cf, @args);
 646
 647     my $err = RRDs::error;
 648     die "RRD error: $err\n" if $err;
 649
 650     die "got wrong time resolution ($step != $reso)\n"
 651         if $step != $reso;
 652
 653     my $res = [];
 654     my $fields = scalar(@$names);
 655     for my $line (@$data) {
 656         my $entry = { 'time' => $start };
 657         $start += $step;
 658         my $found_undefs;
 659         for (my $i = 0; $i < $fields; $i++) {
 660             my $name = $names->[$i];
 661             if (defined(my $val = $line->[$i])) {
 662                 $entry->{$name} = $val;
 663             } else {
 664                 # we only add entryies with all data defined
 665                 # extjs chart has problems with undefined values
 666                 $found_undefs = 1;
 667             }
 668         }
 669         push @$res, $entry if !$found_undefs;
 670     }
 671
 672     return $res;
 673 }
 674
 675 sub create_rrd_graph {
 676     my ($rrdname, $timeframe, $ds, $cf) = @_;
 677
 678     # Using RRD graph is clumsy - maybe it
 679     # is better to simply fetch the data, and do all display
 680     # related things with javascript (new extjs html5 graph library).
 681
 682     my $rrddir = "/var/lib/rrdcached/db";
 683
 684     my $rrd = "$rrddir/$rrdname";
 685
 686     my @ids = PVE::Tools::split_list($ds);
 687
 688     my $ds_txt = join('_', @ids);
 689
 690     my $filename = "${rrd}_${ds_txt}.png";
 691
 692     my $setup = {
 693         hour =>  [ 60, 60 ],
 694         day  =>  [ 60*30, 70 ],
 695         week =>  [ 60*180, 70 ],
 696         month => [ 60*720, 70 ],
 697         year =>  [ 60*10080, 70 ],
 698     };
 699
 700     my ($reso, $count) = @{$setup->{$timeframe}};
 701
 702     my @args = (
 703         "--imgformat" => "PNG",
 704         "--border" => 0,
 705         "--height" => 200,
 706         "--width" => 800,
 707         "--start" => - $reso*$count,
 708         "--end" => 'now' ,
 709         );
 710
 711     my $socket = "/var/run/rrdcached.sock";
 712     push @args, "--daemon" => "unix:$socket" if -S $socket;
 713
 714     my @coldef = ('#00ddff', '#ff0000');
 715
 716     $cf = "AVERAGE" if !$cf;
 717
 718     my $i = 0;
 719     foreach my $id (@ids) {
 720         my $col = $coldef[$i++] || die "fixme: no color definition";
 721         push @args, "DEF:${id}=$rrd:${id}:$cf";
 722         my $dataid = $id;
 723         if ($id eq 'cpu' || $id eq 'iowait') {
 724             push @args, "CDEF:${id}_per=${id},100,*";
 725             $dataid = "${id}_per";
 726         }
 727         push @args, "LINE2:${dataid}${col}:${id}";
 728     }
 729
 730     push @args, '--full-size-mode';
 731
 732     # we do not really store data into the file
 733     my $res = RRDs::graphv('', @args);
 734
 735     my $err = RRDs::error;
 736     die "RRD error: $err\n" if $err;
 737
 738     return { filename => $filename, image => $res->{image} };
 739 }
 740
 741 # a fast way to read files (avoid fuse overhead)
 742 sub get_config {
 743     my ($path) = @_;
 744
 745     return &$ipcc_get_config($path);
 746 }
 747
 748 sub get_cluster_log {
 749     my ($user, $max) = @_;
 750
 751     return &$ipcc_get_cluster_log($user, $max);
 752 }
 753
 754 my $file_info = {};
 755
 756 sub cfs_register_file {
 757     my ($filename, $parser, $writer) = @_;
 758
 759     $observed->{$filename} || die "unknown file '$filename'";
 760
 761     die "file '$filename' already registered" if $file_info->{$filename};
 762
 763     $file_info->{$filename} = {
 764         parser => $parser,
 765         writer => $writer,
 766     };
 767 }
 768
 769 my $ccache_read = sub {
 770     my ($filename, $parser, $version) = @_;
 771
 772     $ccache->{$filename} = {} if !$ccache->{$filename};
 773
 774     my $ci = $ccache->{$filename};
 775
 776     if (!$ci->{version} || !$version || $ci->{version} != $version) {
 777         # we always call the parser, even when the file does not exists
 778         # (in that case $data is undef)
 779         my $data = get_config($filename);
 780         $ci->{data} = &$parser("/etc/pve/$filename", $data);
 781         $ci->{version} = $version;
 782     }
 783
 784     my $res = ref($ci->{data}) ? dclone($ci->{data}) : $ci->{data};
 785
 786     return $res;
 787 };
 788
 789 sub cfs_file_version {
 790     my ($filename) = @_;
 791
 792     my $version;
 793     my $infotag;
 794     if ($filename =~ m!^nodes/[^/]+/(openvz|qemu-server)/(\d+)\.conf$!) {
 795         my ($type, $vmid) = ($1, $2);
 796         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 797             $version = $vmlist->{ids}->{$vmid}->{version};
 798         }
 799         $infotag = "/$type/";
 800     } elsif ($filename =~ m!^nodes/[^/]+/lxc/(\d+)/config$!) {
 801         my $vmid = $1;
 802         if ($vmlist && $vmlist->{ids} && $vmlist->{ids}->{$vmid}) {
 803             $version = $vmlist->{ids}->{$vmid}->{version};
 804         }
 805         $infotag = "/lxc/";
 806     } else {
 807         $infotag = $filename;
 808         $version = $versions->{$filename};
 809     }
 810
 811     my $info = $file_info->{$infotag} ||
 812         die "unknown file type '$filename'\n";
 813
 814     return wantarray ? ($version, $info) : $version;
 815 }
 816
 817 sub cfs_read_file {
 818     my ($filename) = @_;
 819
 820     my ($version, $info) = cfs_file_version($filename);
 821     my $parser = $info->{parser};
 822
 823     return &$ccache_read($filename, $parser, $version);
 824 }
 825
 826 sub cfs_write_file {
 827     my ($filename, $data) = @_;
 828
 829     my ($version, $info) = cfs_file_version($filename);
 830
 831     my $writer = $info->{writer} || die "no writer defined";
 832
 833     my $fsname = "/etc/pve/$filename";
 834
 835     my $raw = &$writer($fsname, $data);
 836
 837     if (my $ci = $ccache->{$filename}) {
 838         $ci->{version} = undef;
 839     }
 840
 841     PVE::Tools::file_set_contents($fsname, $raw);
 842 }
 843
 844 my $cfs_lock = sub {
 845     my ($lockid, $timeout, $code, @param) = @_;
 846
 847     my $res;
 848
 849     # this timeout is for aquire the lock
 850     $timeout = 10 if !$timeout;
 851
 852     my $filename = "$lockdir/$lockid";
 853
 854     my $msg = "can't aquire cfs lock '$lockid'";
 855
 856     eval {
 857
 858         mkdir $lockdir;
 859
 860         if (! -d $lockdir) {
 861             die "$msg: pve cluster filesystem not online.\n";
 862         }
 863
 864         local $SIG{ALRM} = sub { die "got lock request timeout\n"; };
 865
 866         alarm ($timeout);
 867
 868         if (!(mkdir $filename)) {
 869             print STDERR "trying to aquire cfs lock '$lockid' ...";
 870             while (1) {
 871                 if (!(mkdir $filename)) {
 872                     (utime 0, 0, $filename); # cfs unlock request
 873                 } else {
 874                     print STDERR " OK\n";
 875                     last;
 876                 }
 877                 sleep(1);
 878             }
 879         }
 880
 881         # fixed command timeout: cfs locks have a timeout of 120
 882         # using 60 gives us another 60 seconds to abort the task
 883         alarm(60);
 884         local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
 885
 886         cfs_update(); # make sure we read latest versions inside code()
 887
 888         $res = &$code(@param);
 889
 890         alarm(0);
 891     };
 892
 893     my $err = $@;
 894
 895     alarm(0);
 896
 897     if ($err && ($err eq "got lock request timeout\n") &&
 898         !check_cfs_quorum()){
 899         $err = "$msg: no quorum!\n";
 900     }
 901
 902     if (!$err || $err !~ /^got lock timeout -/) {
 903         rmdir $filename; # cfs unlock
 904     }
 905
 906     if ($err) {
 907         $@ = $err;
 908         return undef;
 909     }
 910
 911     $@ = undef;
 912
 913     return $res;
 914 };
 915
 916 sub cfs_lock_file {
 917     my ($filename, $timeout, $code, @param) = @_;
 918
 919     my $info = $observed->{$filename} || die "unknown file '$filename'";
 920
 921     my $lockid = "file-$filename";
 922     $lockid =~ s/[.\/]/_/g;
 923
 924     &$cfs_lock($lockid, $timeout, $code, @param);
 925 }
 926
 927 sub cfs_lock_storage {
 928     my ($storeid, $timeout, $code, @param) = @_;
 929
 930     my $lockid = "storage-$storeid";
 931
 932     &$cfs_lock($lockid, $timeout, $code, @param);
 933 }
 934
 935 my $log_levels = {
 936     "emerg" => 0,
 937     "alert" => 1,
 938     "crit" => 2,
 939     "critical" => 2,
 940     "err" => 3,
 941     "error" => 3,
 942     "warn" => 4,
 943     "warning" => 4,
 944     "notice" => 5,
 945     "info" => 6,
 946     "debug" => 7,
 947 };
 948
 949 sub log_msg {
 950    my ($priority, $ident, $msg) = @_;
 951
 952    if (my $tmp = $log_levels->{$priority}) {
 953        $priority = $tmp;
 954    }
 955
 956    die "need numeric log priority" if $priority !~ /^\d+$/;
 957
 958    my $tag = PVE::SafeSyslog::tag();
 959
 960    $msg = "empty message" if !$msg;
 961
 962    $ident = "" if !$ident;
 963    $ident = encode("ascii", decode_utf8($ident),
 964                    sub { sprintf "\\u%04x", shift });
 965
 966    my $utf8 = decode_utf8($msg);
 967
 968    my $ascii = encode("ascii", $utf8, sub { sprintf "\\u%04x", shift });
 969
 970    if ($ident) {
 971        syslog($priority, "<%s> %s", $ident, $ascii);
 972    } else {
 973        syslog($priority, "%s", $ascii);
 974    }
 975
 976    eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
 977
 978    syslog("err", "writing cluster log failed: $@") if $@;
 979 }
 980
 981 sub check_vmid_unused {
 982     my ($vmid, $noerr) = @_;
 983
 984     my $vmlist = get_vmlist();
 985
 986     my $d = $vmlist->{ids}->{$vmid};
 987     return 1 if !defined($d);
 988
 989     return undef if $noerr;
 990
 991     die "VM $vmid already exists\n" if $d->{type} eq 'qemu';
 992
 993     die "CT $vmid already exists\n";
 994 }
 995
 996 sub check_node_exists {
 997     my ($nodename, $noerr) = @_;
 998
 999     my $nodelist = $clinfo->{nodelist};
1000     return 1 if $nodelist && $nodelist->{$nodename};
1001
1002     return undef if $noerr;
1003
1004     die "no such cluster node '$nodename'\n";
1005 }
1006
1007 # this is also used to get the IP of the local node
1008 sub remote_node_ip {
1009     my ($nodename, $noerr) = @_;
1010
1011     my $nodelist = $clinfo->{nodelist};
1012     if ($nodelist && $nodelist->{$nodename}) {
1013         if (my $ip = $nodelist->{$nodename}->{ip}) {
1014             return $ip if !wantarray;
1015             my $family = $nodelist->{$nodename}->{address_family};
1016             if (!$family) {
1017                 $nodelist->{$nodename}->{address_family} =
1018                     $family =
1019                     PVE::Tools::get_host_address_family($ip);
1020             }
1021             return ($ip, $family);
1022         }
1023     }
1024
1025     # fallback: try to get IP by other means
1026     my ($family, $packed_ip);
1027
1028     eval {
1029         my @res = PVE::Tools::getaddrinfo_all($nodename);
1030         $family = $res[0]->{family};
1031         $packed_ip = (PVE::Tools::unpack_sockaddr_in46($res[0]->{addr}))[2];
1032     };
1033
1034     if ($@) {
1035         die "hostname lookup failed:\n$@" if !$noerr;
1036         return undef;
1037     }
1038
1039     my $ip = Socket::inet_ntop($family, $packed_ip);
1040     if ($ip =~ m/^127\.|^::1$/) {
1041         die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr;
1042         return undef;
1043     }
1044
1045     return wantarray ? ($ip, $family) : $ip;
1046 }
1047
1048 # ssh related utility functions
1049
1050 sub ssh_merge_keys {
1051     # remove duplicate keys in $sshauthkeys
1052     # ssh-copy-id simply add keys, so the file can grow to large
1053
1054     my $data = '';
1055     if (-f $sshauthkeys) {
1056         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
1057         chomp($data);
1058     }
1059
1060     my $found_backup;
1061     if (-f $rootsshauthkeysbackup) {
1062         $data .= "\n";
1063         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
1064         chomp($data);
1065         $found_backup = 1;
1066     }
1067
1068     # always add ourself
1069     if (-f $ssh_rsa_id) {
1070         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
1071         chomp($pub);
1072         $data .= "\n$pub\n";
1073     }
1074
1075     my $newdata = "";
1076     my $vhash = {};
1077     my @lines = split(/\n/, $data);
1078     foreach my $line (@lines) {
1079         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1080             next if $vhash->{$3}++;
1081         }
1082         $newdata .= "$line\n";
1083     }
1084
1085     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
1086
1087     if ($found_backup && -l $rootsshauthkeys) {
1088         # everything went well, so we can remove the backup
1089         unlink $rootsshauthkeysbackup;
1090     }
1091 }
1092
1093 sub setup_sshd_config {
1094
1095     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
1096
1097     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1098
1099     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1100         chomp $conf;
1101         $conf .= "\nPermitRootLogin yes\n";
1102     }
1103
1104     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
1105
1106     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
1107 }
1108
1109 sub setup_rootsshconfig {
1110
1111     # create ssh key if it does not exist
1112     if (! -f $ssh_rsa_id) {
1113         mkdir '/root/.ssh/';
1114         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1115     }
1116
1117     # create ssh config if it does not exist
1118     if (! -f $rootsshconfig) {
1119         mkdir '/root/.ssh';
1120         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
1121             # this is the default ciphers list from debian openssl0.9.8 except blowfish is added as prefered
1122             print $fh "Ciphers blowfish-cbc,aes128-ctr,aes192-ctr,aes256-ctr,arcfour256,arcfour128,aes128-cbc,3des-cbc\n";
1123             close($fh);
1124         }
1125     }
1126 }
1127
1128 sub setup_ssh_keys {
1129
1130     mkdir $authdir;
1131
1132     my $import_ok;
1133
1134     if (! -f $sshauthkeys) {
1135         my $old;
1136         if (-f $rootsshauthkeys) {
1137             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
1138         }
1139         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
1140             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
1141             close($fh);
1142             $import_ok = 1;
1143         }
1144     }
1145
1146     warn "can't create shared ssh key database '$sshauthkeys'\n"
1147         if ! -f $sshauthkeys;
1148
1149     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
1150         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1151             warn "rename $rootsshauthkeys failed - $!\n";
1152         }
1153     }
1154
1155     if (! -l $rootsshauthkeys) {
1156         symlink $sshauthkeys, $rootsshauthkeys;
1157     }
1158
1159     if (! -l $rootsshauthkeys) {
1160         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1161     } else {
1162         unlink $rootsshauthkeysbackup if $import_ok;
1163     }
1164 }
1165
1166 sub ssh_unmerge_known_hosts {
1167     return if ! -l $sshglobalknownhosts;
1168
1169     my $old = '';
1170     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
1171         if -f $sshknownhosts;
1172
1173     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
1174 }
1175
1176 sub ssh_merge_known_hosts {
1177     my ($nodename, $ip_address, $createLink) = @_;
1178
1179     die "no node name specified" if !$nodename;
1180     die "no ip address specified" if !$ip_address;
1181
1182     mkdir $authdir;
1183
1184     if (! -f $sshknownhosts) {
1185         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
1186             close($fh);
1187         }
1188     }
1189
1190     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
1191
1192     my $new = '';
1193
1194     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
1195         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
1196     }
1197
1198     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
1199     die "can't parse $ssh_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/;
1200     $hostkey = $1;
1201
1202     my $data = '';
1203     my $vhash = {};
1204
1205     my $found_nodename;
1206     my $found_local_ip;
1207
1208     my $merge_line = sub {
1209         my ($line, $all) = @_;
1210
1211         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1212             my $key = $1;
1213             my $rsakey = $2;
1214             if (!$vhash->{$key}) {
1215                 $vhash->{$key} = 1;
1216                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1217                     my $salt = decode_base64($1);
1218                     my $digest = $2;
1219                     my $hmac = Digest::HMAC_SHA1->new($salt);
1220                     $hmac->add($nodename);
1221                     my $hd = $hmac->b64digest . '=';
1222                     if ($digest eq $hd) {
1223                         if ($rsakey eq $hostkey) {
1224                             $found_nodename = 1;
1225                             $data .= $line;
1226                         }
1227                         return;
1228                     }
1229                     $hmac = Digest::HMAC_SHA1->new($salt);
1230                     $hmac->add($ip_address);
1231                     $hd = $hmac->b64digest . '=';
1232                     if ($digest eq $hd) {
1233                         if ($rsakey eq $hostkey) {
1234                             $found_local_ip = 1;
1235                             $data .= $line;
1236                         }
1237                         return;
1238                     }
1239                 }
1240                 $data .= $line;
1241             }
1242         } elsif ($all) {
1243             $data .= $line;
1244         }
1245     };
1246
1247     while ($old && $old =~ s/^((.*?)(\n|$))//) {
1248         my $line = "$2\n";
1249         next if $line =~ m/^\s*$/; # skip empty lines
1250         next if $line =~ m/^#/; # skip comments
1251         &$merge_line($line, 1);
1252     }
1253
1254     while ($new && $new =~ s/^((.*?)(\n|$))//) {
1255         my $line = "$2\n";
1256         next if $line =~ m/^\s*$/; # skip empty lines
1257         next if $line =~ m/^#/; # skip comments
1258         &$merge_line($line);
1259     }
1260
1261     my $addIndex = $$;
1262     my $add_known_hosts_entry  = sub {
1263         my ($name, $hostkey) = @_;
1264         $addIndex++;
1265         my $hmac = Digest::HMAC_SHA1->new("$addIndex" . time());
1266         my $b64salt = $hmac->b64digest . '=';
1267         $hmac = Digest::HMAC_SHA1->new(decode_base64($b64salt));
1268         $hmac->add($name);
1269         my $digest = $hmac->b64digest . '=';
1270         $data .= "|1|$b64salt|$digest $hostkey\n";
1271     };
1272
1273     if (!$found_nodename || !$found_local_ip) {
1274         &$add_known_hosts_entry($nodename, $hostkey) if !$found_nodename;
1275         &$add_known_hosts_entry($ip_address, $hostkey) if !$found_local_ip;
1276     }
1277
1278     PVE::Tools::file_set_contents($sshknownhosts, $data);
1279
1280     return if !$createLink;
1281
1282     unlink $sshglobalknownhosts;
1283     symlink $sshknownhosts, $sshglobalknownhosts;
1284
1285     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1286         if ! -l $sshglobalknownhosts;
1287
1288 }
1289
1290 my $datacenter_schema = {
1291     type => "object",
1292     additionalProperties => 0,
1293     properties => {
1294         keyboard => {
1295             optional => 1,
1296             type => 'string',
1297             description => "Default keybord layout for vnc server.",
1298             enum => PVE::Tools::kvmkeymaplist(),
1299         },
1300         language => {
1301             optional => 1,
1302             type => 'string',
1303             description => "Default GUI language.",
1304             enum => [ 'en', 'de' ],
1305         },
1306         http_proxy => {
1307             optional => 1,
1308             type => 'string',
1309             description => "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1310             pattern => "http://.*",
1311         },
1312         migration_unsecure => {
1313             optional => 1,
1314             type => 'boolean',
1315             description => "Migration is secure using SSH tunnel by default. For secure private networks you can disable it to speed up migration.",
1316         },
1317         console => {
1318             optional => 1,
1319             type => 'string',
1320             description => "Select the default Console viewer. You can either use the builtin java applet (VNC), an external virt-viewer comtatible application (SPICE), or an HTML5 based viewer (noVNC).",
1321             enum => ['applet', 'vv', 'html5'],
1322         },
1323         email_from => {
1324             optional => 1,
1325             type => 'string',
1326             format => 'email-opt',
1327             description => "Specify email address to send notification from (default is root@\$hostname)",
1328         },
1329     },
1330 };
1331
1332 # make schema accessible from outside (for documentation)
1333 sub get_datacenter_schema { return $datacenter_schema };
1334
1335 sub parse_datacenter_config {
1336     my ($filename, $raw) = @_;
1337
1338     return PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw);
1339 }
1340
1341 sub write_datacenter_config {
1342     my ($filename, $cfg) = @_;
1343
1344     return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
1345 }
1346
1347 cfs_register_file('datacenter.cfg',
1348                   \&parse_datacenter_config,
1349                   \&write_datacenter_config);
1350
1351 # a very simply parser ...
1352 sub parse_corosync_conf {
1353     my ($filename, $raw) = @_;
1354
1355     return {} if !$raw;
1356
1357     my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
1358
1359     $raw =~ s/#.*$//mg;
1360     $raw =~ s/\r?\n/ /g;
1361     $raw =~ s/\s+/ /g;
1362     $raw =~ s/^\s+//;
1363     $raw =~ s/\s*$//;
1364
1365     my @tokens = split(/\s/, $raw);
1366
1367     my $conf = { section => 'main', children => [] };
1368
1369     my $stack = [];
1370     my $section = $conf;
1371
1372     while (defined(my $token = shift @tokens)) {
1373         my $nexttok = $tokens[0];
1374
1375         if ($nexttok && ($nexttok eq '{')) {
1376             shift @tokens; # skip '{'
1377             my $new_section = {
1378                 section => $token,
1379                 children => [],
1380             };
1381             push @{$section->{children}}, $new_section;
1382             push @$stack, $section;
1383             $section = $new_section;
1384             next;
1385         }
1386
1387         if ($token eq '}') {
1388             $section = pop @$stack;
1389             die "parse error - uncexpected '}'\n" if !$section;
1390             next;
1391         }
1392
1393         my $key = $token;
1394         die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
1395
1396         die "parse error - no value for '$key'\n" if !defined($nexttok);
1397         my $value = shift @tokens;
1398
1399         push @{$section->{children}}, { key => $key, value => $value };
1400     }
1401
1402     $conf->{digest} = $digest;
1403
1404     return $conf;
1405 }
1406
1407 my $dump_corosync_section;
1408 $dump_corosync_section = sub {
1409     my ($section, $prefix) = @_;
1410
1411     my $raw = $prefix . $section->{section} . " {\n";
1412
1413     my @list = grep { defined($_->{key}) } @{$section->{children}};
1414     foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
1415         $raw .= $prefix . "  $child->{key}: $child->{value}\n";
1416     }
1417
1418     @list = grep { defined($_->{section}) } @{$section->{children}};
1419     foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
1420         $raw .= &$dump_corosync_section($child, "$prefix  ");
1421     }
1422
1423     $raw .= $prefix . "}\n\n";
1424
1425     return $raw;
1426
1427 };
1428
1429 sub write_corosync_conf {
1430     my ($filename, $conf) = @_;
1431
1432     my $raw = '';
1433
1434     my $prefix = '';
1435
1436     die "no main section" if $conf->{section} ne 'main';
1437
1438     my @list = grep { defined($_->{key}) } @{$conf->{children}};
1439     foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
1440         $raw .= "$child->{key}: $child->{value}\n";
1441     }
1442
1443     @list = grep { defined($_->{section}) } @{$conf->{children}};
1444     foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
1445         $raw .= &$dump_corosync_section($child, $prefix);
1446     }
1447
1448     return $raw;
1449 }
1450
1451 sub corosync_conf_version {
1452     my ($conf, $noerr, $new_value) = @_;
1453
1454     foreach my $child (@{$conf->{children}}) {
1455         next if !defined($child->{section});
1456         if ($child->{section} eq 'totem') {
1457             foreach my $e (@{$child->{children}}) {
1458                 next if !defined($e->{key});
1459                 if ($e->{key} eq 'config_version') {
1460                     if ($new_value) {
1461                         $e->{value} = $new_value;
1462                         return $new_value;
1463                     } elsif (my $version = int($e->{value})) {
1464                         return $version;
1465                     }
1466                     last;
1467                 }
1468             }
1469         }
1470     }
1471
1472     return undef if $noerr;
1473
1474     die "invalid corosync config - unable to read version\n";
1475 }
1476
1477 # read only - use "rename corosync.conf.new corosync.conf" to write
1478 PVE::Cluster::cfs_register_file('corosync.conf', \&parse_corosync_conf);
1479 # this is read/write
1480 PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_corosync_conf,
1481                                 \&write_corosync_conf);
1482
1483 1;