[pve-manager.git] / lib / PVE.old / Cluster.pm

package PVE::Cluster;

use strict;
use Socket;
use IO::File;
use PVE::Config;
use PVE::Utils;
use PVE::I18N;
use PVE::SafeSyslog;
use Time::HiRes qw (gettimeofday);

my $hostrsapubkey;
my $rootrsapubkey;

# x509 certificate utils

my $basedir = "/etc/pve";
my $pveca_key_fn = "$basedir/priv/pve-root-ca.key";
my $pveca_srl_fn = "$basedir/priv/pve-root-ca.srl";
my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
my $pvessl_key_fn = "$basedir/local/pve-ssl.key";
my $pvessl_cert_fn = "$basedir/local/pve-ssl.pem";

sub gen_local_dirs {
    my ($nodename) = @_;

    (-l "$basedir/local" ) || die "pve configuration filesystem not mounted\n";

    my $dir = "$basedir/nodes/$nodename";
    if (! -d $dir) {
	mkdir($dir) || die "unable to create directory '$dir' - $!\n";
    }
    $dir = "$dir/priv";
    if (! -d $dir) {
	mkdir($dir) || die "unable to create directory '$dir' - $!\n";
    }
}

sub gen_pveca_key {
    
    return if -f $pveca_key_fn;

    eval {
	PVE::Utils::run_command (['openssl', 'genrsa', '-out', $pveca_key_fn, '1024']);
    };

    die "unable to generate pve ca key:\n$@" if $@;
}

sub gen_pveca_cert {
    
    if (-f $pveca_key_fn && -f $pveca_cert_fn) {
	return 0;
    }

    gen_pveca_key();

    # we try to generate an unique 'subject' to avoid browser problems
    # (reused serial numbers, ..)
    my $nid = (split (/\s/, `md5sum '$pveca_key_fn'`))[0] || time();

    eval {
	PVE::Utils::run_command (['openssl', 'req', '-batch', '-days', '3650', '-new',
				  '-x509', '-nodes', '-key', 
				  $pveca_key_fn, '-out', $pveca_cert_fn, '-subj', 
				  "/CN=Proxmox Virtual Environment/OU=$nid/O=PVE Cluster Manager CA/"]);
    };

    die "generating pve root certificate failed:\n$@" if $@;

    return 1;
}

sub gen_pve_ssl_key {
    
    return if -f $pvessl_key_fn;

    eval {
	PVE::Utils::run_command (['openssl', 'genrsa', '-out', $pvessl_key_fn, '1024']);
    };
     
    die "unable to generate pve ssl key:\n$@" if $@;
}

sub update_serial {
    my ($serial) = @_;

    PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
}

sub gen_pve_ssl_cert {
    my ($force, $nodename) = @_;

    return if !$force && -f $pvessl_cert_fn;

    my $names = "IP:127.0.0.1,DNS:localhost";

    my $rc = PVE::Config::read_file ('resolvconf');

    my $packed_ip = gethostbyname($nodename);
    if (defined $packed_ip) {
        my $ip = inet_ntoa($packed_ip);
	$names .= ",IP:" . $ip;
    }

    my $fqdn = $nodename;

    $names .= ",DNS:" . $nodename;

    if ($rc && $rc->{search}) {
	$fqdn = $nodename . "." . $rc->{search};
	$names .= ",DNS:$fqdn";
    }


    my $sslconf = <<__EOD;
RANDFILE = /root/.rnd
extensions = v3_req
 
[ req ]
default_bits = 1024
distinguished_name = req_distinguished_name
req_extensions = v3_req
prompt = no
string_mask = nombstr

[ req_distinguished_name ]
organizationalUnitName = PVE Cluster Node
organizationName = Proxmox Virtual Environment
commonName = $fqdn

[ v3_req ]
basicConstraints = CA:FALSE
nsCertType = server
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = $names
__EOD

    my $cfgfn = "/tmp/pvesslconf-$$.tmp";
    my $fh = IO::File->new ($cfgfn, "w");
    print $fh $sslconf;
    close ($fh);

    my $reqfn = "/tmp/pvecertreq-$$.tmp";
    unlink $reqfn;

    eval {
	PVE::Utils::run_command (['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
				  '-key', $pvessl_key_fn, '-out', $reqfn]);
    };

    if (my $err = $@) {
	unlink $reqfn;
	unlink $cfgfn;
	die "unable to generate pve certificate request:\n$err";
    }

    update_serial ("0000000000000000") if ! -f $pveca_srl_fn;

    eval {
	PVE::Utils::run_command (['openssl', 'x509', '-req', '-in', $reqfn, '-days', '3650', 
				  '-out', $pvessl_cert_fn, '-CAkey', $pveca_key_fn,
				  '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn, 
				  '-extfile', $cfgfn]);
    };

    if (my $err = $@) {
	unlink $reqfn;
	unlink $cfgfn;
	die "unable to generate pve ssl certificate:\n$err";
    }

    unlink $cfgfn;
    unlink $reqfn;
}

sub clusterinfo {
    my ($filename) = @_;

    $filename = "/etc/pve/cluster.cfg" if !$filename;

    my $ifaces = PVE::Config::read_file ("interfaces");
    my $hostname = PVE::Config::read_file ("hostname");
    my $ccfg = PVE::Config::read_file ($filename);

    my $localip = $ifaces->{vmbr0}->{address} || $ifaces->{eth0}->{address};

    my $cinfo;

    if ($ccfg) {
	$cinfo = $ccfg;
	$cinfo->{exists} = 1;
    }

    $cinfo->{local} = {
	role => '-',
	cid => 0,
	ip => $localip,
	name => $hostname,
    };

    my $found = 0;
    foreach my $ni (@{$cinfo->{nodes}}) {
	if ($ni->{ip} eq $localip || $ni->{name} eq $hostname) {
	    $cinfo->{local} = $ni;
	    $found = 1;
	    last;
	}
    }

    if (!$found) {
	push @{$cinfo->{nodes}}, $cinfo->{local};
	$cinfo->{"CID_0"} = $cinfo->{local};
    }
 
    # fixme: assign fixed ports instead?
    # fixme: $ni->{configport} = 50000 + $ni->{cid};
    my $ind = 0;
    foreach my $ni (sort {$a->{cid} <=> $b->{cid}} @{$cinfo->{nodes}}) {
	if ($ni->{cid} == $cinfo->{local}->{cid}) {
	    $ni->{configport} = 83;
	} else {
	    $ni->{configport} = 50000 + $ind;
	    $ind++;
	}
    }

    return $cinfo;
}

sub save_clusterinfo {
    my ($cinfo) = @_;

    my $filename = "/etc/pve/cluster.cfg";

    my $fh = PVE::AtomicFile->open($filename, "w");

    eval {

	return if !$cinfo->{nodes} || scalar (@{$cinfo->{nodes}}) == 0;

	printf ($fh "maxcid $cinfo->{maxcid}\n\n");

	foreach my $ni (@{$cinfo->{nodes}}) {
	    
	    my $cid = $ni->{cid};
	    die "missing cluster id\n" if !$cid;
	    die "missing ip address for node '$cid'\n" if !$ni->{ip};
	    die "missing name for node '$cid'\n" if !$ni->{name};
	    die "missing host RSA key for node '$cid'\n" if !$ni->{hostrsapubkey};
	    die "missing user RSA key for node '$cid'\n" if !$ni->{rootrsapubkey};
	    
	    if ($ni->{role} eq 'M') {
		printf ($fh "master $ni->{cid} {\n");
		printf ($fh " IP: $ni->{ip}\n");
		printf ($fh " NAME: $ni->{name}\n");
		printf ($fh " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n");
		printf ($fh " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n");
		printf ($fh "}\n\n");
	    } elsif ($ni->{role} eq 'N') {
		printf ($fh "node $ni->{cid} {\n");
		printf ($fh " IP: $ni->{ip}\n");
		printf ($fh " NAME: $ni->{name}\n");
		printf ($fh " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n");
		printf ($fh " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n");
		printf ($fh "}\n\n");
	    }
	}
    };

    my $err = $@;
    $fh->detach() if $err;
    $fh->close(1);

    die $err if $err;
}

sub rewrite_keys {
    my ($cinfo) = @_;

    mkdir '/root/.ssh/';

    # rewrite authorized hosts files

    my $filename = '/root/.ssh/authorized_keys';
    my $fh;
    my $changes;

    eval {

	$fh = PVE::AtomicFile->open ($filename, "w");

	my $done = {};

	eval {
	    if (open (ORG, "$filename")) {
		while (my $line = <ORG>) {
		    if ($line =~ m/^\s*ssh-rsa\s+(\S+)\s+root\@(\S+)\s*$/) {
			my ($key, $ip) = ($1, $2);
			my $new;

			foreach my $ni (@{$cinfo->{nodes}}) {
			    $new = $ni if $ni->{ip} eq $ip;
			}

			if ($new) {
			    if (!$done->{$ip}) {
				$changes = 1 if $key ne $new->{rootrsapubkey};
				printf ($fh "ssh-rsa %s root\@%s\n", $new->{rootrsapubkey}, $new->{ip});
				$done->{$ip} = 1;
			    }
			} else {
			    print $fh $line; # copy line to new file
			}
		    } else {
			print $fh $line; # copy line to new file
		    }
		}
		close (ORG);
	    }
	};

	foreach my $ni (@{$cinfo->{nodes}}) {
	    if (!$done->{$ni->{ip}}) {
		$changes = 1;
		printf ($fh "ssh-rsa %s root\@%s\n", $ni->{rootrsapubkey}, $ni->{ip});
		$done->{$ni->{ip}} = 1;
	    }
	}
    };

    $fh->close() if $fh;

    chmod (0600, $filename);

    # rewrite known hosts files

    $filename = '/root/.ssh/known_hosts';

    eval {
	
	$fh = PVE::AtomicFile->open($filename, "w");

	my $done = {};

	eval {
	    if (open (ORG, "$filename")) {
		while (my $line = <ORG>) {
		    if ($line =~ m/^\s*(\S+)\s+ssh-rsa\s+(\S+)\s*$/) {
			my ($ip, $key) = ($1, $2);
			my $new;

			foreach my $ni (@{$cinfo->{nodes}}) {
			    $new = $ni if $ni->{ip} eq $ip;
			}

			if ($new) {
			    if (!$done->{$ip}) {
				$changes = 1 if $key ne $new->{hostrsapubkey};
				printf ($fh "%s ssh-rsa %s\n", $new->{ip}, $new->{hostrsapubkey});
				$done->{$ip} = 1;
			    }
			} else {
			    print $fh $line; # copy line to new file
			}
		    } else {
			print $fh $line; # copy line to new file
		    }
		}
		close (ORG);
	    }
	};

	foreach my $ni (@{$cinfo->{nodes}}) {
	    if (!$done->{$ni->{ip}}) {
		$changes = 1;
		printf ($fh "%s ssh-rsa %s\n", $ni->{ip}, $ni->{hostrsapubkey});
		$done->{$ni->{ip}} = 1;
	    }
	}
    };

    $fh->close() if $fh;
 
    return $changes;
}

sub cluster_sync_mastercfg {
    my ($cinfo, $syncip, $noreload) = @_;

    my $lip = $cinfo->{local}->{ip};
    my $lname = $cinfo->{local}->{name};

    my $cmpccfg;
    my $cmppvecfg;
    my $cmpqemucfg;
    my $cmpvzdump;
    my $cmpstoragecfg;

    my $storagecfg_old = PVE::Config::read_file ('storagecfg');

    if ($syncip ne $lip) {

	mkdir '/etc/pve/master';
	unlink </etc/pve/master/*>;

	my $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq', 
		   "$syncip:/etc/pve/* /etc/cron.d/vzdump", '/etc/pve/master/',
		   '--exclude', '*~' ];

	eval {
	    my $out = PVE::Utils::run_command ($cmd);
	};

	my $err = $@;

	if ($err) {
	    my $cmdtxt = join (' ', @$cmd);
	    die "syncing master configuration from '$syncip' failed ($cmdtxt) : $err\n";
	}
	
	# verify that the remote host is cluster master

	my $newcinfo = clusterinfo ('/etc/pve/master/cluster.cfg');
	
	if (!$newcinfo->{master} || ($newcinfo->{master}->{ip} ne $syncip)) {
	    die "host '$syncip' is not cluster master\n";
	}

	if ($newcinfo->{local}->{role} ne 'N') {
	    syslog ('info', "local host is no longer part of cluster '$syncip'");
	    rename '/etc/pve/master/cluster.cfg', '/etc/pve/cluster.cfg';
	    die "local host is no node of cluster '$syncip' " .
		"(role = $newcinfo->{local}->{role})\n";
	}

	# we are part of the cluster

	$cmpccfg = (system ("cmp -s /etc/pve/master/cluster.cfg /etc/pve/cluster.cfg") != 0)
	    if -f '/etc/pve/master/cluster.cfg';

	rename '/etc/pve/master/cluster.cfg', '/etc/pve/cluster.cfg' if $cmpccfg;

	# check for storage changes

	if (-f '/etc/pve/master/storage.cfg') {
	    $cmpstoragecfg = (system ("cmp -s /etc/pve/master/storage.cfg /etc/pve/storage.cfg") != 0);
	    rename '/etc/pve/master/storage.cfg', '/etc/pve/storage.cfg' if $cmpstoragecfg;
	} else {
	    unlink '/etc/pve/storage.cfg';
	}

	# check for vzdump crontab changes

	$cmpvzdump = (system ("cmp -s /etc/pve/master/vzdump /etc/cron.d/vzdump") != 0)
	    if -f '/etc/pve/master/vzdump';

	# check for CA cerificate change
	if ((-f '/etc/pve/master/pve-root-ca.pem') && (-f '/etc/pve/master/pve-root-ca.key') &&
	    (system ("cmp -s /etc/pve/master/pve-root-ca.pem /etc/pve/pve-root-ca.pem") != 0)) {
	    rename '/etc/pve/master/pve-root-ca.pem', '/etc/pve/pve-root-ca.pem';
	    rename '/etc/pve/master/pve-root-ca.key', '/etc/pve/pve-root-ca.key';
	    my $serial = sprintf ("%04X000000000000", $newcinfo->{local}->{cid});
	    update_serial ($serial);
	    eval {
		# make sure we have a private key
		gen_pve_ssl_key();
		# force key rewrite
		gen_pve_ssl_cert (1, $newcinfo);
	    };
	    my $err = $@;
	    if ($err) {
		syslog ('err', "pve key generation failed - try 'pcecert' manually");
	    }
	} 

	$cmppvecfg = (system ("cmp -s /etc/pve/master/pve.cfg /etc/pve/pve.cfg") != 0)
	    if -f '/etc/pve/master/pve.cfg';

	rename '/etc/pve/master/pve.cfg', '/etc/pve/pve.cfg' if $cmppvecfg;

	$cmpqemucfg = (system ("cmp -s /etc/pve/master/qemu-server.cfg /etc/pve/qemu-server.cfg") != 0)
	    if -f '/etc/pve/master/qemu-server.cfg';

	rename '/etc/pve/master/qemu-server.cfg', '/etc/pve/qemu-server.cfg' if $cmpqemucfg;

	#fixme: store/remove additional files

    }

    if ($cmpccfg ||  # cluster info changed 
	($syncip eq $lip)) { # or forced sync withe proxca -s

	if ($cmpccfg) {
	    syslog ('info', "detected changed cluster config");
	}

	$cinfo = clusterinfo ();

	my $changes = rewrite_keys ($cinfo);

	if ($changes) {
	    PVE::Utils::service_cmd ('sshd', 'reload');
	}

	PVE::Utils::service_cmd ('pvetunnel', 'reload') if !$noreload;	
    }

    if ($cmppvecfg) { # pve.cfg settings changed

	# fixme: implement me

    }

    if ($cmpqemucfg) { # qemu-server.cfg settings changed
	# nothing to do
    }

    if ($cmpvzdump) {
	syslog ('info', "installing new vzdump crontab");
	rename '/etc/pve/master/vzdump', '/etc/cron.d/vzdump';
    }

    if ($cmpstoragecfg) {
	my $storagecfg_new = PVE::Config::read_file ('storagecfg');

	foreach my $sid (PVE::Storage::storage_ids ($storagecfg_old)) {
	    my $ocfg = PVE::Storage::storage_config ($storagecfg_old, $sid);
	    if (my $ncfg = PVE::Storage::storage_config ($storagecfg_new, $sid, 1)) {
		if (!$ocfg->{disable} && $ncfg->{disable}) {
		    syslog ('info', "deactivate storage '$sid'");
		    eval { PVE::Storage::deactivate_storage ($storagecfg_new, $sid); };
		    syslog ('err', $@) if $@;
		}
	    } else {
		if (!$ocfg->{disable}) {
		    syslog ('info', "deactivate removed storage '$sid'");
		    eval { PVE::Storage::deactivate_storage ($storagecfg_old, $sid); };
		    syslog ('err', $@) if $@;   
		}
	    }
	}
    }
}

sub vzlist_update {
    my ($cid, $ticket) = @_;
    
    my $cinfo = clusterinfo ();

    my $vzlist;

    my $cvzl;

    my $conn = PVE::ConfigClient::connect ($ticket);

    my $ni;
    if (($ni = $cinfo->{"CID_$cid"})) {
	my $rcon = PVE::ConfigClient::connect ($ticket, $cinfo, $cid);
	$vzlist = $rcon->vzlist()->result;
    }

    if ($vzlist) {
	$cvzl = $conn->cluster_vzlist($cid, $vzlist)->result;
    }

    return $cvzl;
}

sub load_vmconfig {
    my ($cinfo, $cid, $veid, $type, $ticket) = @_;

    my $remcon = PVE::ConfigClient::connect ($ticket, $cinfo, $cid);
    my $vminfo = $remcon->vmconfig ($veid, $type)->result;
    
    if (!$vminfo) {
	die "unable to get configuration data for VEID '$veid'";
    }

    $vminfo->{ni} = $cinfo->{"CID_$cid"};

    return $vminfo;
}

sub sync_templates {
    my ($cinfo) = @_;

    if ($cinfo->{master} && ($cinfo->{master}->{cid} !=  $cinfo->{local}->{cid})) {

	my $remip = $cinfo->{master}->{ip};

	my $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-aq', 
		   '--delete', '--bwlimit=10240',
		   "$remip:/var/lib/vz/template", "/var/lib/vz" ];

	eval { PVE::Utils::run_command ($cmd); };

	my $err = $@;

	if ($err) {
	    my $cmdtxt = join (' ', @$cmd);
	    die "syncing template from master '$remip' failed ($cmdtxt) : $err\n";
	}
    }
}

sub get_nextid {
    my ($vzlist, $vmops) = @_; 

    my $veexist = {};
    
    PVE::Utils::foreach_vmrec ($vzlist, sub {
	my ($cid, $vmid) = @_;
	$veexist->{$1} = 1;
    });

    PVE::Utils::foreach_vmrec ($vmops, sub {
	my ($cid, $vmid, $d) = @_;
	next if $d->{command} ne 'create';
	$veexist->{$1} = 1;
    });

    my $nextveid;
    for (my $i = 101; $i < 10000; $i++) {
	if (!$veexist->{$i}) {
	    $nextveid = $i;
	    last;
	}
    }

    return $nextveid;
}

1;
Commit	Line	Data
aff192e6 DM	1	package PVE::Cluster;
	2
	3	use strict;
	4	use Socket;
	5	use IO::File;
	6	use PVE::Config;
	7	use PVE::Utils;
	8	use PVE::I18N;
	9	use PVE::SafeSyslog;
	10	use Time::HiRes qw (gettimeofday);
	11
	12	my $hostrsapubkey;
	13	my $rootrsapubkey;
	14
	15	# x509 certificate utils
	16
	17	my $basedir = "/etc/pve";
	18	my $pveca_key_fn = "$basedir/priv/pve-root-ca.key";
	19	my $pveca_srl_fn = "$basedir/priv/pve-root-ca.srl";
	20	my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
	21	my $pvessl_key_fn = "$basedir/local/pve-ssl.key";
	22	my $pvessl_cert_fn = "$basedir/local/pve-ssl.pem";
	23
	24	sub gen_local_dirs {
	25	my ($nodename) = @_;
	26
	27	(-l "$basedir/local" ) \|\| die "pve configuration filesystem not mounted\n";
	28
	29	my $dir = "$basedir/nodes/$nodename";
	30	if (! -d $dir) {
	31	mkdir($dir) \|\| die "unable to create directory '$dir' - $!\n";
	32	}
	33	$dir = "$dir/priv";
	34	if (! -d $dir) {
	35	mkdir($dir) \|\| die "unable to create directory '$dir' - $!\n";
	36	}
	37	}
	38
	39	sub gen_pveca_key {
	40
	41	return if -f $pveca_key_fn;
	42
	43	eval {
	44	PVE::Utils::run_command (['openssl', 'genrsa', '-out', $pveca_key_fn, '1024']);
	45	};
	46
	47	die "unable to generate pve ca key:\n$@" if $@;
	48	}
	49
	50	sub gen_pveca_cert {
	51
	52	if (-f $pveca_key_fn && -f $pveca_cert_fn) {
	53	return 0;
	54	}
	55
	56	gen_pveca_key();
	57
	58	# we try to generate an unique 'subject' to avoid browser problems
	59	# (reused serial numbers, ..)
	60	my $nid = (split (/\s/, `md5sum '$pveca_key_fn'`))[0] \|\| time();
	61
	62	eval {
	63	PVE::Utils::run_command (['openssl', 'req', '-batch', '-days', '3650', '-new',
	64	'-x509', '-nodes', '-key',
65	$pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
66	"/CN=Proxmox Virtual Environment/OU=$nid/O=PVE Cluster Manager CA/"]);
67	};
68
69	die "generating pve root certificate failed:\n$@" if $@;
70
71	return 1;
72	}
73
74	sub gen_pve_ssl_key {
75
76	return if -f $pvessl_key_fn;
77
78	eval {
79	PVE::Utils::run_command (['openssl', 'genrsa', '-out', $pvessl_key_fn, '1024']);
80	};
81
82	die "unable to generate pve ssl key:\n$@" if $@;
83	}
84
85	sub update_serial {
86	my ($serial) = @_;
87
88	PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
89	}
90
91	sub gen_pve_ssl_cert {
92	my ($force, $nodename) = @_;
93
94	return if !$force && -f $pvessl_cert_fn;
95
96	my $names = "IP:127.0.0.1,DNS:localhost";
97
98	my $rc = PVE::Config::read_file ('resolvconf');
99
100	my $packed_ip = gethostbyname($nodename);
101	if (defined $packed_ip) {
102	my $ip = inet_ntoa($packed_ip);
103	$names .= ",IP:" . $ip;
104	}
105
106	my $fqdn = $nodename;
107
108	$names .= ",DNS:" . $nodename;
109
110	if ($rc && $rc->{search}) {
111	$fqdn = $nodename . "." . $rc->{search};
112	$names .= ",DNS:$fqdn";
113	}
114
115
116	my $sslconf = <<__EOD;
117	RANDFILE = /root/.rnd
118	extensions = v3_req
119
120	[ req ]
121	default_bits = 1024
122	distinguished_name = req_distinguished_name
123	req_extensions = v3_req
124	prompt = no
125	string_mask = nombstr
126
127	[ req_distinguished_name ]
128	organizationalUnitName = PVE Cluster Node
129	organizationName = Proxmox Virtual Environment
130	commonName = $fqdn
131
132	[ v3_req ]
133	basicConstraints = CA:FALSE
134	nsCertType = server
135	keyUsage = nonRepudiation, digitalSignature, keyEncipherment
136	subjectAltName = $names
137	__EOD
138
139	my $cfgfn = "/tmp/pvesslconf-$$.tmp";
140	my $fh = IO::File->new ($cfgfn, "w");
141	print $fh $sslconf;
142	close ($fh);
143
144	my $reqfn = "/tmp/pvecertreq-$$.tmp";
145	unlink $reqfn;
146
147	eval {
148	PVE::Utils::run_command (['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
149	'-key', $pvessl_key_fn, '-out', $reqfn]);
150	};
151
152	if (my $err = $@) {
153	unlink $reqfn;
154	unlink $cfgfn;
155	die "unable to generate pve certificate request:\n$err";
156	}
157
158	update_serial ("0000000000000000") if ! -f $pveca_srl_fn;
159
160	eval {
161	PVE::Utils::run_command (['openssl', 'x509', '-req', '-in', $reqfn, '-days', '3650',
162	'-out', $pvessl_cert_fn, '-CAkey', $pveca_key_fn,
163	'-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn,
164	'-extfile', $cfgfn]);
165	};
166
167	if (my $err = $@) {
168	unlink $reqfn;
169	unlink $cfgfn;
170	die "unable to generate pve ssl certificate:\n$err";
171	}
172
173	unlink $cfgfn;
174	unlink $reqfn;
175	}
176
177	sub clusterinfo {
178	my ($filename) = @_;
179
180	$filename = "/etc/pve/cluster.cfg" if !$filename;
181
182	my $ifaces = PVE::Config::read_file ("interfaces");
183	my $hostname = PVE::Config::read_file ("hostname");
184	my $ccfg = PVE::Config::read_file ($filename);
185
186	my $localip = $ifaces->{vmbr0}->{address} \|\| $ifaces->{eth0}->{address};
187
188	my $cinfo;
189
190	if ($ccfg) {
191	$cinfo = $ccfg;
192	$cinfo->{exists} = 1;
193	}
194
195	$cinfo->{local} = {
196	role => '-',
197	cid => 0,
198	ip => $localip,
199	name => $hostname,
200	};
201
202	my $found = 0;
203	foreach my $ni (@{$cinfo->{nodes}}) {
204	if ($ni->{ip} eq $localip \|\| $ni->{name} eq $hostname) {
205	$cinfo->{local} = $ni;
206	$found = 1;
207	last;
208	}
209	}
210
211	if (!$found) {
212	push @{$cinfo->{nodes}}, $cinfo->{local};
213	$cinfo->{"CID_0"} = $cinfo->{local};
214	}
215
216	# fixme: assign fixed ports instead?
217	# fixme: $ni->{configport} = 50000 + $ni->{cid};
218	my $ind = 0;
219	foreach my $ni (sort {$a->{cid} <=> $b->{cid}} @{$cinfo->{nodes}}) {
220	if ($ni->{cid} == $cinfo->{local}->{cid}) {
221	$ni->{configport} = 83;
222	} else {
223	$ni->{configport} = 50000 + $ind;
224	$ind++;
225	}
226	}
227
228	return $cinfo;
229	}
230
231	sub save_clusterinfo {
232	my ($cinfo) = @_;
233
234	my $filename = "/etc/pve/cluster.cfg";
235
236	my $fh = PVE::AtomicFile->open($filename, "w");
237
238	eval {
239
240	return if !$cinfo->{nodes} \|\| scalar (@{$cinfo->{nodes}}) == 0;
241
242	printf ($fh "maxcid $cinfo->{maxcid}\n\n");
243
244	foreach my $ni (@{$cinfo->{nodes}}) {
245
246	my $cid = $ni->{cid};
247	die "missing cluster id\n" if !$cid;
248	die "missing ip address for node '$cid'\n" if !$ni->{ip};
249	die "missing name for node '$cid'\n" if !$ni->{name};
250	die "missing host RSA key for node '$cid'\n" if !$ni->{hostrsapubkey};
251	die "missing user RSA key for node '$cid'\n" if !$ni->{rootrsapubkey};
252
253	if ($ni->{role} eq 'M') {
254	printf ($fh "master $ni->{cid} {\n");
255	printf ($fh " IP: $ni->{ip}\n");
256	printf ($fh " NAME: $ni->{name}\n");
257	printf ($fh " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n");
258	printf ($fh " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n");
259	printf ($fh "}\n\n");
260	} elsif ($ni->{role} eq 'N') {
261	printf ($fh "node $ni->{cid} {\n");
262	printf ($fh " IP: $ni->{ip}\n");
263	printf ($fh " NAME: $ni->{name}\n");
264	printf ($fh " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n");
265	printf ($fh " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n");
266	printf ($fh "}\n\n");
267	}
268	}
269	};
270
271	my $err = $@;
272	$fh->detach() if $err;
273	$fh->close(1);
274
275	die $err if $err;
276	}
277
278	sub rewrite_keys {
279	my ($cinfo) = @_;
280
281	mkdir '/root/.ssh/';
282
283	# rewrite authorized hosts files
284
285	my $filename = '/root/.ssh/authorized_keys';
286	my $fh;
287	my $changes;
288
289	eval {
290
291	$fh = PVE::AtomicFile->open ($filename, "w");
292
293	my $done = {};
294
295	eval {
296	if (open (ORG, "$filename")) {
297	while (my $line = <ORG>) {
298	if ($line =~ m/^\sssh-rsa\s+(\S+)\s+root\@(\S+)\s$/) {
299	my ($key, $ip) = ($1, $2);
300	my $new;
301
302	foreach my $ni (@{$cinfo->{nodes}}) {
303	$new = $ni if $ni->{ip} eq $ip;
304	}
305
306	if ($new) {
307	if (!$done->{$ip}) {
308	$changes = 1 if $key ne $new->{rootrsapubkey};
309	printf ($fh "ssh-rsa %s root\@%s\n", $new->{rootrsapubkey}, $new->{ip});
310	$done->{$ip} = 1;
311	}
312	} else {
313	print $fh $line; # copy line to new file
314	}
315	} else {
316	print $fh $line; # copy line to new file
317	}
318	}
319	close (ORG);
320	}
321	};
322
323	foreach my $ni (@{$cinfo->{nodes}}) {
324	if (!$done->{$ni->{ip}}) {
325	$changes = 1;
326	printf ($fh "ssh-rsa %s root\@%s\n", $ni->{rootrsapubkey}, $ni->{ip});
327	$done->{$ni->{ip}} = 1;
328	}
329	}
330	};
331
332	$fh->close() if $fh;
333
334	chmod (0600, $filename);
335
336	# rewrite known hosts files
337
338	$filename = '/root/.ssh/known_hosts';
339
340	eval {
341
342	$fh = PVE::AtomicFile->open($filename, "w");
343
344	my $done = {};
345
346	eval {
347	if (open (ORG, "$filename")) {
348	while (my $line = <ORG>) {
349	if ($line =~ m/^\s(\S+)\s+ssh-rsa\s+(\S+)\s$/) {
350	my ($ip, $key) = ($1, $2);
351	my $new;
352
353	foreach my $ni (@{$cinfo->{nodes}}) {
354	$new = $ni if $ni->{ip} eq $ip;
355	}
356
357	if ($new) {
358	if (!$done->{$ip}) {
359	$changes = 1 if $key ne $new->{hostrsapubkey};
360	printf ($fh "%s ssh-rsa %s\n", $new->{ip}, $new->{hostrsapubkey});
361	$done->{$ip} = 1;
362	}
363	} else {
364	print $fh $line; # copy line to new file
365	}
366	} else {
367	print $fh $line; # copy line to new file
368	}
369	}
370	close (ORG);
371	}
372	};
373
374	foreach my $ni (@{$cinfo->{nodes}}) {
375	if (!$done->{$ni->{ip}}) {
376	$changes = 1;
377	printf ($fh "%s ssh-rsa %s\n", $ni->{ip}, $ni->{hostrsapubkey});
378	$done->{$ni->{ip}} = 1;
379	}
380	}
381	};
382
383	$fh->close() if $fh;
384
385	return $changes;
386	}
387
388	sub cluster_sync_mastercfg {
389	my ($cinfo, $syncip, $noreload) = @_;
390
391	my $lip = $cinfo->{local}->{ip};
392	my $lname = $cinfo->{local}->{name};
393
394	my $cmpccfg;
395	my $cmppvecfg;
396	my $cmpqemucfg;
397	my $cmpvzdump;
398	my $cmpstoragecfg;
399
400	my $storagecfg_old = PVE::Config::read_file ('storagecfg');
401
402	if ($syncip ne $lip) {
403
404	mkdir '/etc/pve/master';
405	unlink </etc/pve/master/*>;
406
407	my $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
408	"$syncip:/etc/pve/* /etc/cron.d/vzdump", '/etc/pve/master/',
409	'--exclude', '*~' ];
410
411	eval {
412	my $out = PVE::Utils::run_command ($cmd);
413	};
414
415	my $err = $@;
416
417	if ($err) {
418	my $cmdtxt = join (' ', @$cmd);
419	die "syncing master configuration from '$syncip' failed ($cmdtxt) : $err\n";
420	}
421
422	# verify that the remote host is cluster master
423
424	my $newcinfo = clusterinfo ('/etc/pve/master/cluster.cfg');
425
426	if (!$newcinfo->{master} \|\| ($newcinfo->{master}->{ip} ne $syncip)) {
427	die "host '$syncip' is not cluster master\n";
428	}
429
430	if ($newcinfo->{local}->{role} ne 'N') {
431	syslog ('info', "local host is no longer part of cluster '$syncip'");
432	rename '/etc/pve/master/cluster.cfg', '/etc/pve/cluster.cfg';
433	die "local host is no node of cluster '$syncip' " .
434	"(role = $newcinfo->{local}->{role})\n";
435	}
436
437	# we are part of the cluster
438
439	$cmpccfg = (system ("cmp -s /etc/pve/master/cluster.cfg /etc/pve/cluster.cfg") != 0)
440	if -f '/etc/pve/master/cluster.cfg';
441
442	rename '/etc/pve/master/cluster.cfg', '/etc/pve/cluster.cfg' if $cmpccfg;
443
444	# check for storage changes
445
446	if (-f '/etc/pve/master/storage.cfg') {
447	$cmpstoragecfg = (system ("cmp -s /etc/pve/master/storage.cfg /etc/pve/storage.cfg") != 0);
448	rename '/etc/pve/master/storage.cfg', '/etc/pve/storage.cfg' if $cmpstoragecfg;
449	} else {
450	unlink '/etc/pve/storage.cfg';
451	}
452
453	# check for vzdump crontab changes
454
455	$cmpvzdump = (system ("cmp -s /etc/pve/master/vzdump /etc/cron.d/vzdump") != 0)
456	if -f '/etc/pve/master/vzdump';
457
458	# check for CA cerificate change
459	if ((-f '/etc/pve/master/pve-root-ca.pem') && (-f '/etc/pve/master/pve-root-ca.key') &&
460	(system ("cmp -s /etc/pve/master/pve-root-ca.pem /etc/pve/pve-root-ca.pem") != 0)) {
461	rename '/etc/pve/master/pve-root-ca.pem', '/etc/pve/pve-root-ca.pem';
462	rename '/etc/pve/master/pve-root-ca.key', '/etc/pve/pve-root-ca.key';
463	my $serial = sprintf ("%04X000000000000", $newcinfo->{local}->{cid});
464	update_serial ($serial);
465	eval {
466	# make sure we have a private key
467	gen_pve_ssl_key();
468	# force key rewrite
469	gen_pve_ssl_cert (1, $newcinfo);
470	};
471	my $err = $@;
472	if ($err) {
473	syslog ('err', "pve key generation failed - try 'pcecert' manually");
474	}
475	}
476
477	$cmppvecfg = (system ("cmp -s /etc/pve/master/pve.cfg /etc/pve/pve.cfg") != 0)
478	if -f '/etc/pve/master/pve.cfg';
479
480	rename '/etc/pve/master/pve.cfg', '/etc/pve/pve.cfg' if $cmppvecfg;
481
482	$cmpqemucfg = (system ("cmp -s /etc/pve/master/qemu-server.cfg /etc/pve/qemu-server.cfg") != 0)
483	if -f '/etc/pve/master/qemu-server.cfg';
484
485	rename '/etc/pve/master/qemu-server.cfg', '/etc/pve/qemu-server.cfg' if $cmpqemucfg;
486
487	#fixme: store/remove additional files
488
489	}
490
491	if ($cmpccfg \|\| # cluster info changed
492	($syncip eq $lip)) { # or forced sync withe proxca -s
493
494	if ($cmpccfg) {
495	syslog ('info', "detected changed cluster config");
496	}
497
498	$cinfo = clusterinfo ();
499
500	my $changes = rewrite_keys ($cinfo);
501
502	if ($changes) {
503	PVE::Utils::service_cmd ('sshd', 'reload');
504	}
505
506	PVE::Utils::service_cmd ('pvetunnel', 'reload') if !$noreload;
507	}
508
509	if ($cmppvecfg) { # pve.cfg settings changed
510
511	# fixme: implement me
512
513	}
514
515	if ($cmpqemucfg) { # qemu-server.cfg settings changed
516	# nothing to do
517	}
518
519	if ($cmpvzdump) {
520	syslog ('info', "installing new vzdump crontab");
521	rename '/etc/pve/master/vzdump', '/etc/cron.d/vzdump';
522	}
523
524	if ($cmpstoragecfg) {
525	my $storagecfg_new = PVE::Config::read_file ('storagecfg');
526
527	foreach my $sid (PVE::Storage::storage_ids ($storagecfg_old)) {
528	my $ocfg = PVE::Storage::storage_config ($storagecfg_old, $sid);
529	if (my $ncfg = PVE::Storage::storage_config ($storagecfg_new, $sid, 1)) {
530	if (!$ocfg->{disable} && $ncfg->{disable}) {
531	syslog ('info', "deactivate storage '$sid'");
532	eval { PVE::Storage::deactivate_storage ($storagecfg_new, $sid); };
533	syslog ('err', $@) if $@;
534	}
535	} else {
536	if (!$ocfg->{disable}) {
537	syslog ('info', "deactivate removed storage '$sid'");
538	eval { PVE::Storage::deactivate_storage ($storagecfg_old, $sid); };
539	syslog ('err', $@) if $@;
540	}
541	}
542	}
543	}
544	}
545
546	sub vzlist_update {
547	my ($cid, $ticket) = @_;
548
549	my $cinfo = clusterinfo ();
550
551	my $vzlist;
552
553	my $cvzl;
554
555	my $conn = PVE::ConfigClient::connect ($ticket);
556
557	my $ni;
558	if (($ni = $cinfo->{"CID_$cid"})) {
559	my $rcon = PVE::ConfigClient::connect ($ticket, $cinfo, $cid);
560	$vzlist = $rcon->vzlist()->result;
561	}
562
563	if ($vzlist) {
564	$cvzl = $conn->cluster_vzlist($cid, $vzlist)->result;
565	}
566
567	return $cvzl;
568	}
569
570	sub load_vmconfig {
571	my ($cinfo, $cid, $veid, $type, $ticket) = @_;
572
573	my $remcon = PVE::ConfigClient::connect ($ticket, $cinfo, $cid);
574	my $vminfo = $remcon->vmconfig ($veid, $type)->result;
575
576	if (!$vminfo) {
577	die "unable to get configuration data for VEID '$veid'";
578	}
579
580	$vminfo->{ni} = $cinfo->{"CID_$cid"};
581
582	return $vminfo;
583	}
584
585	sub sync_templates {
586	my ($cinfo) = @_;
587
588	if ($cinfo->{master} && ($cinfo->{master}->{cid} != $cinfo->{local}->{cid})) {
589
590	my $remip = $cinfo->{master}->{ip};
591
592	my $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-aq',
593	'--delete', '--bwlimit=10240',
594	"$remip:/var/lib/vz/template", "/var/lib/vz" ];
595
596	eval { PVE::Utils::run_command ($cmd); };
597
598	my $err = $@;
599
600	if ($err) {
601	my $cmdtxt = join (' ', @$cmd);
602	die "syncing template from master '$remip' failed ($cmdtxt) : $err\n";
603	}
604	}
605	}
606
607	sub get_nextid {
608	my ($vzlist, $vmops) = @_;
609
610	my $veexist = {};
611
612	PVE::Utils::foreach_vmrec ($vzlist, sub {
613	my ($cid, $vmid) = @_;
614	$veexist->{$1} = 1;
615	});
616
617	PVE::Utils::foreach_vmrec ($vmops, sub {
618	my ($cid, $vmid, $d) = @_;
619	next if $d->{command} ne 'create';
620	$veexist->{$1} = 1;
621	});
622
623	my $nextveid;
624	for (my $i = 101; $i < 10000; $i++) {
625	if (!$veexist->{$i}) {
626	$nextveid = $i;
627	last;
628	}
629	}
630
631	return $nextveid;
632	}
633
634	1;