From: Dietmar Maurer Date: Tue, 21 Feb 2017 08:14:14 +0000 (+0100) Subject: PMG/ClusterConfig.pm: better code to read/write cluster config X-Git-Url: https://git.proxmox.com/?a=commitdiff_plain;h=9f67f5b3e94157de4523c22f83ed27aa4556b99d;p=pmg-api.git PMG/ClusterConfig.pm: better code to read/write cluster config using SectionConfig. --- diff --git a/Makefile b/Makefile index 1bb8e32..295463c 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ LIBSOURCES = \ PMG/NoVncIndex.pm \ PMG/Config.pm \ PMG/Cluster.pm \ + PMG/ClusterConfig.pm \ PMG/HTTPServer.pm \ PMG/Ticket.pm \ PMG/AccessControl.pm \ diff --git a/PMG/CLI/pmgdb.pm b/PMG/CLI/pmgdb.pm index 0d045c8..da4c74b 100644 --- a/PMG/CLI/pmgdb.pm +++ b/PMG/CLI/pmgdb.pm @@ -12,6 +12,7 @@ use PVE::CLIHandler; use PMG::DBTools; use PMG::RuleDB; use PMG::Cluster; +use PMG::ClusterConfig; use PMG::Statistic; use base qw(PVE::CLIHandler); diff --git a/PMG/Cluster.pm b/PMG/Cluster.pm index 7b44d93..5678d7d 100644 --- a/PMG/Cluster.pm +++ b/PMG/Cluster.pm @@ -7,38 +7,14 @@ use Socket; use PVE::Tools; use PVE::INotify; -# this is also used to get the IP of the local node -sub lookup_node_ip { - my ($nodename, $noerr) = @_; - - my ($family, $packed_ip); - - eval { - my @res = PVE::Tools::getaddrinfo_all($nodename); - $family = $res[0]->{family}; - $packed_ip = (PVE::Tools::unpack_sockaddr_in46($res[0]->{addr}))[2]; - }; - - if ($@) { - die "hostname lookup failed:\n$@" if !$noerr; - return undef; - } - - my $ip = Socket::inet_ntop($family, $packed_ip); - if ($ip =~ m/^127\.|^::1$/) { - die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr; - return undef; - } - - return wantarray ? ($ip, $family) : $ip; -} +use PMG::ClusterConfig; sub remote_node_ip { my ($nodename, $noerr) = @_; my $cinfo = PVE::INotify::read_file("cluster.conf"); - foreach my $entry (@{$cinfo->{nodes}}) { + foreach my $entry (@{$cinfo->{ids}}) { if ($entry->{name} eq $nodename) { my $ip = $entry->{ip}; return $ip if !wantarray; @@ -48,13 +24,13 @@ sub remote_node_ip { } # fallback: try to get IP by other means - return lookup_node_ip($nodename, $noerr); + return PMG::Utils::lookup_node_ip($nodename, $noerr); } sub get_master_node { my ($cinfo) = @_; - $cinfo //= PVE::INotify::read_file("cluster.conf"); + $cinfo = PVE::INotify::read_file("cluster.conf"); return $cinfo->{master}->{name} if defined($cinfo->{master}); @@ -171,165 +147,4 @@ sub check_cert_fingerprint { return 0; } -sub read_cluster_conf { - my ($filename, $fh) = @_; - - my $localname = PVE::INotify::nodename(); - my $localip = lookup_node_ip($localname); - - my $level = 0; - my $maxcid = 0; - - my $cinfo; - - $cinfo->{nodes} = []; - $cinfo->{remnodes} = []; - - $cinfo->{local} = { - role => '-', - cid => 0, - ip => $localip, - name => $localname, - configport => 83, - dbport => 5432, - }; - - # fixme: add test is local node is part of node list - if (defined($fh)) { - - $cinfo->{exists} = 1; # cluster configuratin file exists and is readable - - while (defined(my $line = <$fh>)) { - chomp $line; - - next if $line =~ m/^\s*$/; # skip empty lines - - if ($line =~ m/^maxcid\s+(\d+)\s*$/i) { - $maxcid = $1 > $maxcid ? $1 : $maxcid; - next; - } - - if ($line =~ m/^(master|node)\s+(\d+)\s+\{\s*$/i) { - $level++; - my ($t, $cid) = (lc($1), $2); - - $maxcid = $cid > $maxcid ? $cid : $maxcid; - - my $res = { - role => $t eq 'master' ? 'M' : 'N', - cid => $cid - }; - - while (defined($line = <$fh>)) { - chomp $line; - next if $line =~ m/^\s*$/; # skip empty lines - if ($line =~ m/^\}\s*$/) { - $level--; - last; - } - - if ($line =~ m/^\s*(\S+)\s*:\s*(\S+)\s*$/) { - my ($n, $v) = (lc $1, $2); - - # fixme: do syntax checks - if ($n eq 'ip') { - $res->{$n} = $v; - } elsif ($n eq 'name') { - $res->{$n} = $v; - } elsif ($n eq 'hostrsapubkey') { - $res->{$n} = $v; - } elsif ($n eq 'rootrsapubkey') { - $res->{$n} = $v; - } else { - die "syntax error in configuration file\n"; - } - } else { - die "syntax error in configuration file\n"; - } - } - - die "missing ip address for node '$cid'\n" if !$res->{ip}; - die "missing name for node '$cid'\n" if !$res->{name}; - #die "missing host RSA key for node '$cid'\n" if !$res->{hostrsapubkey}; - #die "missing user RSA key for node '$cid'\n" if !$res->{rootrsapubkey}; - - push @{$cinfo->{nodes}}, $res; - - if ($res->{role} eq 'M') { - $cinfo->{master} = $res; - } - - if ($res->{ip} eq $localname) { - $cinfo->{local} = $res; - } - } else { - die "syntax error in configuration file\n"; - } - } - } - - die "syntax error in configuration file\n" if $level; - - $cinfo->{maxcid} = $maxcid; - - my @cidlist = (); - foreach my $ni (@{$cinfo->{nodes}}) { - next if $cinfo->{local}->{cid} == $ni->{cid}; # skip local CID - push @cidlist, $ni->{cid}; - } - - my $ind = 0; - my $portid = {}; - foreach my $cid (sort @cidlist) { - $portid->{$cid} = $ind; - $ind++; - } - - foreach my $ni (@{$cinfo->{nodes}}) { - # fixme: do we still need those ports? - $ni->{configport} = $ni->{cid} == $cinfo->{local}->{cid} ? 83 : 50000 + $portid->{$ni->{cid}}; - $ni->{dbport} = $ni->{cid} == $cinfo->{local}->{cid} ? 5432 : 50100 + $portid->{$ni->{cid}}; - } - - foreach my $ni (@{$cinfo->{nodes}}) { - next if $ni->{cid} == $cinfo->{local}->{cid}; - push @{$cinfo->{remnodes}}, $ni->{cid}; - } - - return $cinfo; -} - -sub write_cluster_conf { - my ($filename, $fh, $cinfo) = @_; - - my $raw = "maxcid $cinfo->{maxcid}\n\n"; - - foreach my $ni (@{$cinfo->{nodes}}) { - - if ($ni->{role} eq 'M') { - $raw .= "master $ni->{cid} {\n"; - $raw .= " IP: $ni->{ip}\n"; - $raw .= " NAME: $ni->{name}\n"; - $raw .= " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n"; - $raw .= " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n"; - $raw .= "}\n\n"; - } elsif ($ni->{role} eq 'N') { - $raw .= "node $ni->{cid} {\n"; - $raw .= " IP: $ni->{ip}\n"; - $raw .= " NAME: $ni->{name}\n"; - $raw .= " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n"; - $raw .= " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n"; - $raw .= "}\n\n"; - } - } - - PVE::Tools::safe_print($filename, $fh, $raw); -} - -PVE::INotify::register_file('cluster.conf', "/etc/proxmox/cluster.conf", - \&read_cluster_conf, - \&write_cluster_conf, - undef, - always_call_parser => 1); - 1; diff --git a/PMG/ClusterConfig.pm b/PMG/ClusterConfig.pm new file mode 100644 index 0000000..6a2425e --- /dev/null +++ b/PMG/ClusterConfig.pm @@ -0,0 +1,233 @@ +package PMG::ClusterConfig::Base; + +use strict; +use warnings; +use Data::Dumper; + +use PVE::Tools; +use PVE::JSONSchema qw(get_standard_option); +use PVE::SectionConfig; + +use base qw(PVE::SectionConfig); + +my $defaultData = { + propertyList => { + type => { description => "Cluster node type." }, + cid => { + description => "Cluster Node ID.", + type => 'integer', + minimum => 1, + }, + }, +}; + +sub private { + return $defaultData; +} + +sub parse_section_header { + my ($class, $line) = @_; + + if ($line =~ m/^(node|master):\s*(\d+)\s*$/) { + my ($type, $sectionId) = ($1, $2); + my $errmsg = undef; # set if you want to skip whole section + my $config = {}; # to return additional attributes + return ($type, $sectionId, $errmsg, $config); + } + return undef; +} + + +package PMG::ClusterConfig::Node; + +use strict; +use warnings; + +use base qw(PMG::ClusterConfig::Base); + +sub type { + return 'node'; +} +sub properties { + return { + ip => { + description => "IP address.", + type => 'string', format => 'address', + }, + name => { + description => "Node name.", + type => 'string', format =>'pve-node', + }, + hostrsapubkey => { + description => "Public SSH RSA key for the host.", + type => 'string', + }, + rootrsapubkey => { + description => "Public SSH RSA key for the root user.", + type => 'string', + }, + }; +} + +sub options { + return { + ip => { fixed => 1 }, + name => { fixed => 1 }, + hostrsapubkey => {}, + rootrsapubkey => {}, + }; +} + +package PMG::ClusterConfig::Master; + +use strict; +use warnings; + +use base qw(PMG::ClusterConfig::Base); + +sub type { + return 'master'; +} + +sub properties { + return { + maxcid => { + description => "Maximum used cluster node ID (used internally, do not modify).", + type => 'integer', + minimum => 1, + }, + }; +} + +sub options { + return { + maxcid => { fixed => 1 }, + ip => { fixed => 1 }, + name => { fixed => 1 }, + hostrsapubkey => {}, + rootrsapubkey => {}, + }; +} + +package PMG::ClusterConfig; + +use strict; +use warnings; +use Data::Dumper; + +use PVE::SafeSyslog; +use PVE::Tools; +use PVE::INotify; + +use PMG::Utils; + +PMG::ClusterConfig::Node->register; +PMG::ClusterConfig::Master->register; +PMG::ClusterConfig::Base->init(); + + +sub new { + my ($type) = @_; + + my $class = ref($type) || $type; + + my $cfg = PVE::INotify::read_file("cluster.conf"); + + return bless $cfg, $class; +} + +sub write { + my ($self) = @_; + + PVE::INotify::write_file("cluster.conf", $self); +} + +my $lockfile = "/var/lock/pmgcluster.lck"; + +sub lock_config { + my ($code, $errmsg) = @_; + + my $p = PVE::Tools::lock_file($lockfile, undef, $code); + if (my $err = $@) { + $errmsg ? die "$errmsg: $err" : die $err; + } +} + +sub read_cluster_conf { + my ($filename, $fh) = @_; + + local $/ = undef; # slurp mode + + my $raw = defined($fh) ? <$fh> : undef; + + my $cinfo = PMG::ClusterConfig::Base->parse_config($filename, $raw); + print Dumper($cinfo); + + my $localname = PVE::INotify::nodename(); + my $localip = PMG::Utils::lookup_node_ip($localname); + + $cinfo->{remnodes} = []; + $cinfo->{configport}->{0} = 83; + $cinfo->{dbport}->{0} = 5432; + + $cinfo->{local} = { + cid => 0, + ip => $localip, + name => $localname, + }; + + my $maxcid = 0; + my $names_hash = {}; + + my $errprefix = "unable to parse $filename"; + + foreach my $cid (keys %{$cinfo->{ids}}) { + my $d = $cinfo->{ids}->{$cid}; + + die "$errprefix: duplicate use of name '$d->{name}'\n" if $names_hash->{$d->{name}}; + $names_hash->{$d->{name}} = 1; + + $d->{cid} = $cid; + $maxcid = $cid > $maxcid ? $cid : $maxcid; + $maxcid = $d->{maxcid} if defined($d->{maxcid}) && $d->{maxcid} > $maxcid; + $cinfo->{master} = $d if $d->{type} eq 'master'; + $cinfo->{'local'} = $d if $d->{name} eq $localname; + } + + if ($maxcid) { + die "$errprefix: cluster without master node\n" + if !defined($cinfo->{master}); + $cinfo->{master}->{maxcid} = $maxcid; + } + + my $ind = 0; + foreach my $cid (sort keys %{$cinfo->{ids}}) { + if ($cinfo->{'local'}->{cid} == $cid) { # local CID + $cinfo->{configport}->{$cid} = 83; + $cinfo->{dbport}->{$cid} = 5432; + } else { + my $portid = $ind++; + $cinfo->{configport}->{$cid} = 50000 + $portid; + $cinfo->{dbport}->{$cid} = 50100 + $portid; + push @{$cinfo->{remnodes}}, $cinfo->{ids}->{$cid}; + } + } + + return $cinfo; +} + +sub write_cluster_conf { + my ($filename, $fh, $cfg) = @_; + + my $raw = PMG::ClusterConfig::Base->write_config($filename, $cfg); + + PVE::Tools::safe_print($filename, $fh, $raw); +} + +PVE::INotify::register_file('cluster.conf', "/etc/proxmox/cluster.conf", + \&read_cluster_conf, + \&write_cluster_conf, + undef, + always_call_parser => 1); + +1; diff --git a/PMG/Statistic.pm b/PMG/Statistic.pm index b3065d9..9609469 100755 --- a/PMG/Statistic.pm +++ b/PMG/Statistic.pm @@ -9,6 +9,7 @@ use Time::Zone; use PVE::SafeSyslog; +use PMG::ClusterConfig; use PMG::RuleDB; sub new { @@ -129,8 +130,8 @@ sub update_stats_generic { sub update_stats_dailystat { my ($dbh, $cinfo) = @_; - my $role = $cinfo->{local}->{role}; - return 0 if !(($role eq '-') || ($role eq 'M')); + my $role = $cinfo->{local}->{type} // '-'; + return 0 if !(($role eq '-') || ($role eq 'master')); my $select = "SELECT sub.*, dailystat.time IS NOT NULL as exists FROM " . "(SELECT COUNT (CASE WHEN direction THEN 1 ELSE NULL END) as count_in, " . @@ -202,8 +203,8 @@ sub update_stats_dailystat { sub update_stats_domainstat_in { my ($dbh, $cinfo) = @_; - my $role = $cinfo->{local}->{role}; - return 0 if !(($role eq '-') || ($role eq 'M')); + my $role = $cinfo->{local}->{type} // '-'; + return 0 if !(($role eq '-') || ($role eq 'master')); my $sub1 = "select distinct cstatistic_cid, cstatistic_rid, " . "lower(substring(receiver from position ('\@' in receiver) + 1)) as domain, " . @@ -266,8 +267,8 @@ sub update_stats_domainstat_in { sub update_stats_domainstat_out { my ($dbh, $cinfo) = @_; - my $role = $cinfo->{local}->{role}; - return 0 if !(($role eq '-') || ($role eq 'M')); + my $role = $cinfo->{local}->{type} // '-'; + return 0 if !(($role eq '-') || ($role eq 'master')); my $select = "SELECT sub.*, domainstat.time IS NOT NULL as exists FROM " . "(SELECT COUNT (ID) as count_out, SUM (bytes) / (1024.0*1024) as bytes_out, " . @@ -324,8 +325,8 @@ sub update_stats_domainstat_out { sub update_stats_virusinfo { my ($dbh, $cinfo) = @_; - my $role = $cinfo->{local}->{role}; - return 0 if !(($role eq '-') || ($role eq 'M')); + my $role = $cinfo->{local}->{type} // '-'; + return 0 if !(($role eq '-') || ($role eq 'master')); my $select = "SELECT sub.*, virusinfo.time IS NOT NULL as exists FROM " . "(SELECT ((cstatistic.time + __timezone__) / 86400) * 86400 as day, " . diff --git a/PMG/Utils.pm b/PMG/Utils.pm index e7e7cc0..a04518d 100644 --- a/PMG/Utils.pm +++ b/PMG/Utils.pm @@ -15,6 +15,7 @@ use Time::HiRes qw (gettimeofday); use Xdgmime; use Data::Dumper; use Net::IP; +use Socket; use PVE::Network; use PVE::Tools; @@ -401,4 +402,30 @@ sub service_cmd { PVE::Tools::run_command(['systemctl', $cmd, $service]); }; +# this is also used to get the IP of the local node +sub lookup_node_ip { + my ($nodename, $noerr) = @_; + + my ($family, $packed_ip); + + eval { + my @res = PVE::Tools::getaddrinfo_all($nodename); + $family = $res[0]->{family}; + $packed_ip = (PVE::Tools::unpack_sockaddr_in46($res[0]->{addr}))[2]; + }; + + if ($@) { + die "hostname lookup failed:\n$@" if !$noerr; + return undef; + } + + my $ip = Socket::inet_ntop($family, $packed_ip); + if ($ip =~ m/^127\.|^::1$/) { + die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr; + return undef; + } + + return wantarray ? ($ip, $family) : $ip; +} + 1; diff --git a/bin/pmg-smtp-filter b/bin/pmg-smtp-filter index 026c1e8..25fda9f 100755 --- a/bin/pmg-smtp-filter +++ b/bin/pmg-smtp-filter @@ -28,6 +28,7 @@ use Mail::SpamAssassin::NetSet; use PMG::pmgcfg; use PMG::Utils; use PMG::Cluster; +use PMG::ClusterConfig; use PMG::DBTools; use PMG::RuleDB;