]>
git.proxmox.com Git - pve-manager.git/blob - lib/PVE.old/Cluster.pm
10 use Time
::HiRes qw
(gettimeofday
);
15 # x509 certificate utils
17 my $basedir = "/etc/pve";
18 my $pveca_key_fn = "$basedir/priv/pve-root-ca.key";
19 my $pveca_srl_fn = "$basedir/priv/pve-root-ca.srl";
20 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
21 my $pvessl_key_fn = "$basedir/local/pve-ssl.key";
22 my $pvessl_cert_fn = "$basedir/local/pve-ssl.pem";
27 (-l
"$basedir/local" ) || die "pve configuration filesystem not mounted\n";
29 my $dir = "$basedir/nodes/$nodename";
31 mkdir($dir) || die "unable to create directory '$dir' - $!\n";
35 mkdir($dir) || die "unable to create directory '$dir' - $!\n";
41 return if -f
$pveca_key_fn;
44 PVE
::Utils
::run_command
(['openssl', 'genrsa', '-out', $pveca_key_fn, '1024']);
47 die "unable to generate pve ca key:\n$@" if $@;
52 if (-f
$pveca_key_fn && -f
$pveca_cert_fn) {
58 # we try to generate an unique 'subject' to avoid browser problems
59 # (reused serial numbers, ..)
60 my $nid = (split (/\s/, `md5sum '$pveca_key_fn'`))[0] || time();
63 PVE
::Utils
::run_command
(['openssl', 'req', '-batch', '-days', '3650', '-new',
64 '-x509', '-nodes', '-key',
65 $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
66 "/CN=Proxmox Virtual Environment/OU=$nid/O=PVE Cluster Manager CA/"]);
69 die "generating pve root certificate failed:\n$@" if $@;
76 return if -f
$pvessl_key_fn;
79 PVE
::Utils
::run_command
(['openssl', 'genrsa', '-out', $pvessl_key_fn, '1024']);
82 die "unable to generate pve ssl key:\n$@" if $@;
88 PVE
::Tools
::file_set_contents
($pveca_srl_fn, $serial);
91 sub gen_pve_ssl_cert
{
92 my ($force, $nodename) = @_;
94 return if !$force && -f
$pvessl_cert_fn;
96 my $names = "IP:127.0.0.1,DNS:localhost";
98 my $rc = PVE
::Config
::read_file
('resolvconf');
100 my $packed_ip = gethostbyname($nodename);
101 if (defined $packed_ip) {
102 my $ip = inet_ntoa
($packed_ip);
103 $names .= ",IP:" . $ip;
106 my $fqdn = $nodename;
108 $names .= ",DNS:" . $nodename;
110 if ($rc && $rc->{search
}) {
111 $fqdn = $nodename . "." . $rc->{search
};
112 $names .= ",DNS:$fqdn";
116 my $sslconf = <<__EOD;
117 RANDFILE = /root/.rnd
122 distinguished_name = req_distinguished_name
123 req_extensions = v3_req
125 string_mask = nombstr
127 [ req_distinguished_name ]
128 organizationalUnitName = PVE Cluster Node
129 organizationName = Proxmox Virtual Environment
133 basicConstraints = CA:FALSE
135 keyUsage = nonRepudiation, digitalSignature, keyEncipherment
136 subjectAltName = $names
139 my $cfgfn = "/tmp/pvesslconf-$$.tmp";
140 my $fh = IO
::File-
>new ($cfgfn, "w");
144 my $reqfn = "/tmp/pvecertreq-$$.tmp";
148 PVE
::Utils
::run_command
(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
149 '-key', $pvessl_key_fn, '-out', $reqfn]);
155 die "unable to generate pve certificate request:\n$err";
158 update_serial
("0000000000000000") if ! -f
$pveca_srl_fn;
161 PVE
::Utils
::run_command
(['openssl', 'x509', '-req', '-in', $reqfn, '-days', '3650',
162 '-out', $pvessl_cert_fn, '-CAkey', $pveca_key_fn,
163 '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn,
164 '-extfile', $cfgfn]);
170 die "unable to generate pve ssl certificate:\n$err";
180 $filename = "/etc/pve/cluster.cfg" if !$filename;
182 my $ifaces = PVE
::Config
::read_file
("interfaces");
183 my $hostname = PVE
::Config
::read_file
("hostname");
184 my $ccfg = PVE
::Config
::read_file
($filename);
186 my $localip = $ifaces->{vmbr0
}->{address
} || $ifaces->{eth0
}->{address
};
192 $cinfo->{exists} = 1;
203 foreach my $ni (@{$cinfo->{nodes
}}) {
204 if ($ni->{ip
} eq $localip || $ni->{name
} eq $hostname) {
205 $cinfo->{local} = $ni;
212 push @{$cinfo->{nodes
}}, $cinfo->{local};
213 $cinfo->{"CID_0"} = $cinfo->{local};
216 # fixme: assign fixed ports instead?
217 # fixme: $ni->{configport} = 50000 + $ni->{cid};
219 foreach my $ni (sort {$a->{cid
} <=> $b->{cid
}} @{$cinfo->{nodes
}}) {
220 if ($ni->{cid
} == $cinfo->{local}->{cid
}) {
221 $ni->{configport
} = 83;
223 $ni->{configport
} = 50000 + $ind;
231 sub save_clusterinfo
{
234 my $filename = "/etc/pve/cluster.cfg";
236 my $fh = PVE
::AtomicFile-
>open($filename, "w");
240 return if !$cinfo->{nodes
} || scalar (@{$cinfo->{nodes
}}) == 0;
242 printf ($fh "maxcid $cinfo->{maxcid}\n\n");
244 foreach my $ni (@{$cinfo->{nodes
}}) {
246 my $cid = $ni->{cid
};
247 die "missing cluster id\n" if !$cid;
248 die "missing ip address for node '$cid'\n" if !$ni->{ip
};
249 die "missing name for node '$cid'\n" if !$ni->{name
};
250 die "missing host RSA key for node '$cid'\n" if !$ni->{hostrsapubkey
};
251 die "missing user RSA key for node '$cid'\n" if !$ni->{rootrsapubkey
};
253 if ($ni->{role} eq 'M') {
254 printf ($fh "master $ni->{cid} {\n");
255 printf ($fh " IP: $ni->{ip}\n");
256 printf ($fh " NAME: $ni->{name}\n");
257 printf ($fh " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n");
258 printf ($fh " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n");
259 printf ($fh "}\n\n");
260 } elsif ($ni->{role} eq 'N') {
261 printf ($fh "node $ni->{cid} {\n");
262 printf ($fh " IP: $ni->{ip}\n");
263 printf ($fh " NAME: $ni->{name}\n");
264 printf ($fh " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n");
265 printf ($fh " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n");
266 printf ($fh "}\n\n");
272 $fh->detach() if $err;
283 # rewrite authorized hosts files
285 my $filename = '/root/.ssh/authorized_keys';
291 $fh = PVE
::AtomicFile-
>open ($filename, "w");
296 if (open (ORG
, "$filename")) {
297 while (my $line = <ORG
>) {
298 if ($line =~ m/^\s*ssh-rsa\s+(\S+)\s+root\@(\S+)\s*$/) {
299 my ($key, $ip) = ($1, $2);
302 foreach my $ni (@{$cinfo->{nodes
}}) {
303 $new = $ni if $ni->{ip
} eq $ip;
308 $changes = 1 if $key ne $new->{rootrsapubkey
};
309 printf ($fh "ssh-rsa %s root\@%s\n", $new->{rootrsapubkey
}, $new->{ip
});
313 print $fh $line; # copy line to new file
316 print $fh $line; # copy line to new file
323 foreach my $ni (@{$cinfo->{nodes
}}) {
324 if (!$done->{$ni->{ip
}}) {
326 printf ($fh "ssh-rsa %s root\@%s\n", $ni->{rootrsapubkey
}, $ni->{ip
});
327 $done->{$ni->{ip
}} = 1;
334 chmod (0600, $filename);
336 # rewrite known hosts files
338 $filename = '/root/.ssh/known_hosts';
342 $fh = PVE
::AtomicFile-
>open($filename, "w");
347 if (open (ORG
, "$filename")) {
348 while (my $line = <ORG
>) {
349 if ($line =~ m/^\s*(\S+)\s+ssh-rsa\s+(\S+)\s*$/) {
350 my ($ip, $key) = ($1, $2);
353 foreach my $ni (@{$cinfo->{nodes
}}) {
354 $new = $ni if $ni->{ip
} eq $ip;
359 $changes = 1 if $key ne $new->{hostrsapubkey
};
360 printf ($fh "%s ssh-rsa %s\n", $new->{ip
}, $new->{hostrsapubkey
});
364 print $fh $line; # copy line to new file
367 print $fh $line; # copy line to new file
374 foreach my $ni (@{$cinfo->{nodes
}}) {
375 if (!$done->{$ni->{ip
}}) {
377 printf ($fh "%s ssh-rsa %s\n", $ni->{ip
}, $ni->{hostrsapubkey
});
378 $done->{$ni->{ip
}} = 1;
388 sub cluster_sync_mastercfg
{
389 my ($cinfo, $syncip, $noreload) = @_;
391 my $lip = $cinfo->{local}->{ip
};
392 my $lname = $cinfo->{local}->{name
};
400 my $storagecfg_old = PVE
::Config
::read_file
('storagecfg');
402 if ($syncip ne $lip) {
404 mkdir '/etc/pve/master';
405 unlink </etc/pve
/master/*>;
407 my $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
408 "$syncip:/etc/pve/* /etc/cron.d/vzdump", '/etc/pve/master/',
412 my $out = PVE
::Utils
::run_command
($cmd);
418 my $cmdtxt = join (' ', @$cmd);
419 die "syncing master configuration from '$syncip' failed ($cmdtxt) : $err\n";
422 # verify that the remote host is cluster master
424 my $newcinfo = clusterinfo
('/etc/pve/master/cluster.cfg');
426 if (!$newcinfo->{master
} || ($newcinfo->{master
}->{ip
} ne $syncip)) {
427 die "host '$syncip' is not cluster master\n";
430 if ($newcinfo->{local}->{role} ne 'N') {
431 syslog
('info', "local host is no longer part of cluster '$syncip'");
432 rename '/etc/pve/master/cluster.cfg', '/etc/pve/cluster.cfg';
433 die "local host is no node of cluster '$syncip' " .
434 "(role = $newcinfo->{local}->{role})\n";
437 # we are part of the cluster
439 $cmpccfg = (system ("cmp -s /etc/pve/master/cluster.cfg /etc/pve/cluster.cfg") != 0)
440 if -f
'/etc/pve/master/cluster.cfg';
442 rename '/etc/pve/master/cluster.cfg', '/etc/pve/cluster.cfg' if $cmpccfg;
444 # check for storage changes
446 if (-f
'/etc/pve/master/storage.cfg') {
447 $cmpstoragecfg = (system ("cmp -s /etc/pve/master/storage.cfg /etc/pve/storage.cfg") != 0);
448 rename '/etc/pve/master/storage.cfg', '/etc/pve/storage.cfg' if $cmpstoragecfg;
450 unlink '/etc/pve/storage.cfg';
453 # check for vzdump crontab changes
455 $cmpvzdump = (system ("cmp -s /etc/pve/master/vzdump /etc/cron.d/vzdump") != 0)
456 if -f
'/etc/pve/master/vzdump';
458 # check for CA cerificate change
459 if ((-f
'/etc/pve/master/pve-root-ca.pem') && (-f
'/etc/pve/master/pve-root-ca.key') &&
460 (system ("cmp -s /etc/pve/master/pve-root-ca.pem /etc/pve/pve-root-ca.pem") != 0)) {
461 rename '/etc/pve/master/pve-root-ca.pem', '/etc/pve/pve-root-ca.pem';
462 rename '/etc/pve/master/pve-root-ca.key', '/etc/pve/pve-root-ca.key';
463 my $serial = sprintf ("%04X000000000000", $newcinfo->{local}->{cid
});
464 update_serial
($serial);
466 # make sure we have a private key
469 gen_pve_ssl_cert
(1, $newcinfo);
473 syslog
('err', "pve key generation failed - try 'pcecert' manually");
477 $cmppvecfg = (system ("cmp -s /etc/pve/master/pve.cfg /etc/pve/pve.cfg") != 0)
478 if -f
'/etc/pve/master/pve.cfg';
480 rename '/etc/pve/master/pve.cfg', '/etc/pve/pve.cfg' if $cmppvecfg;
482 $cmpqemucfg = (system ("cmp -s /etc/pve/master/qemu-server.cfg /etc/pve/qemu-server.cfg") != 0)
483 if -f
'/etc/pve/master/qemu-server.cfg';
485 rename '/etc/pve/master/qemu-server.cfg', '/etc/pve/qemu-server.cfg' if $cmpqemucfg;
487 #fixme: store/remove additional files
491 if ($cmpccfg || # cluster info changed
492 ($syncip eq $lip)) { # or forced sync withe proxca -s
495 syslog
('info', "detected changed cluster config");
498 $cinfo = clusterinfo
();
500 my $changes = rewrite_keys
($cinfo);
503 PVE
::Utils
::service_cmd
('sshd', 'reload');
506 PVE
::Utils
::service_cmd
('pvetunnel', 'reload') if !$noreload;
509 if ($cmppvecfg) { # pve.cfg settings changed
511 # fixme: implement me
515 if ($cmpqemucfg) { # qemu-server.cfg settings changed
520 syslog
('info', "installing new vzdump crontab");
521 rename '/etc/pve/master/vzdump', '/etc/cron.d/vzdump';
524 if ($cmpstoragecfg) {
525 my $storagecfg_new = PVE
::Config
::read_file
('storagecfg');
527 foreach my $sid (PVE
::Storage
::storage_ids
($storagecfg_old)) {
528 my $ocfg = PVE
::Storage
::storage_config
($storagecfg_old, $sid);
529 if (my $ncfg = PVE
::Storage
::storage_config
($storagecfg_new, $sid, 1)) {
530 if (!$ocfg->{disable
} && $ncfg->{disable
}) {
531 syslog
('info', "deactivate storage '$sid'");
532 eval { PVE
::Storage
::deactivate_storage
($storagecfg_new, $sid); };
533 syslog
('err', $@) if $@;
536 if (!$ocfg->{disable
}) {
537 syslog
('info', "deactivate removed storage '$sid'");
538 eval { PVE
::Storage
::deactivate_storage
($storagecfg_old, $sid); };
539 syslog
('err', $@) if $@;
547 my ($cid, $ticket) = @_;
549 my $cinfo = clusterinfo
();
555 my $conn = PVE
::ConfigClient
::connect ($ticket);
558 if (($ni = $cinfo->{"CID_$cid"})) {
559 my $rcon = PVE
::ConfigClient
::connect ($ticket, $cinfo, $cid);
560 $vzlist = $rcon->vzlist()->result;
564 $cvzl = $conn->cluster_vzlist($cid, $vzlist)->result;
571 my ($cinfo, $cid, $veid, $type, $ticket) = @_;
573 my $remcon = PVE
::ConfigClient
::connect ($ticket, $cinfo, $cid);
574 my $vminfo = $remcon->vmconfig ($veid, $type)->result;
577 die "unable to get configuration data for VEID '$veid'";
580 $vminfo->{ni
} = $cinfo->{"CID_$cid"};
588 if ($cinfo->{master
} && ($cinfo->{master
}->{cid
} != $cinfo->{local}->{cid
})) {
590 my $remip = $cinfo->{master
}->{ip
};
592 my $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-aq',
593 '--delete', '--bwlimit=10240',
594 "$remip:/var/lib/vz/template", "/var/lib/vz" ];
596 eval { PVE
::Utils
::run_command
($cmd); };
601 my $cmdtxt = join (' ', @$cmd);
602 die "syncing template from master '$remip' failed ($cmdtxt) : $err\n";
608 my ($vzlist, $vmops) = @_;
612 PVE
::Utils
::foreach_vmrec
($vzlist, sub {
613 my ($cid, $vmid) = @_;
617 PVE
::Utils
::foreach_vmrec
($vmops, sub {
618 my ($cid, $vmid, $d) = @_;
619 next if $d->{command
} ne 'create';
624 for (my $i = 101; $i < 10000; $i++) {
625 if (!$veexist->{$i}) {