]>
Commit | Line | Data |
---|---|---|
13d44dc5 DM |
1 | package PVE::CLI::pvecm; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | use Getopt::Long; | |
6 | use Socket; | |
7 | use IO::File; | |
d56a1ff3 WB |
8 | use IO::Socket::IP; |
9 | use POSIX; | |
13d44dc5 DM |
10 | use Net::IP; |
11 | use File::Path; | |
12 | use File::Basename; | |
c53b111f | 13 | use Data::Dumper; # fixme: remove |
13d44dc5 DM |
14 | use PVE::Tools; |
15 | use PVE::Cluster; | |
16 | use PVE::INotify; | |
17 | use PVE::JSONSchema; | |
13d44dc5 | 18 | use PVE::CLIHandler; |
b6973a89 | 19 | use PVE::Corosync; |
13d44dc5 DM |
20 | |
21 | use base qw(PVE::CLIHandler); | |
22 | ||
23 | $ENV{HOME} = '/root'; # for ssh-copy-id | |
24 | ||
25 | my $basedir = "/etc/pve"; | |
26 | my $clusterconf = "$basedir/corosync.conf"; | |
27 | my $libdir = "/var/lib/pve-cluster"; | |
28 | my $backupdir = "/var/lib/pve-cluster/backup"; | |
29 | my $dbfile = "$libdir/config.db"; | |
30 | my $authfile = "/etc/corosync/authkey"; | |
31 | ||
32 | sub backup_database { | |
33 | ||
34 | print "backup old database\n"; | |
35 | ||
36 | mkdir $backupdir; | |
c53b111f | 37 | |
13d44dc5 | 38 | my $ctime = time(); |
de4b4155 FG |
39 | my $cmd = [ |
40 | ['echo', '.dump'], | |
41 | ['sqlite3', $dbfile], | |
7eee3d64 | 42 | ['gzip', '-', \ ">${backupdir}/config-${ctime}.sql.gz"], |
de4b4155 | 43 | ]; |
13d44dc5 | 44 | |
0c17b64f | 45 | PVE::Tools::run_command($cmd, 'errmsg' => "cannot backup old database\n"); |
13d44dc5 DM |
46 | |
47 | # purge older backup | |
48 | my $maxfiles = 10; | |
49 | ||
50 | my @bklist = (); | |
51 | foreach my $fn (<$backupdir/config-*.sql.gz>) { | |
52 | if ($fn =~ m!/config-(\d+)\.sql.gz$!) { | |
53 | push @bklist, [$fn, $1]; | |
54 | } | |
55 | } | |
c53b111f | 56 | |
13d44dc5 DM |
57 | @bklist = sort { $b->[1] <=> $a->[1] } @bklist; |
58 | ||
59 | while (scalar (@bklist) >= $maxfiles) { | |
60 | my $d = pop @bklist; | |
61 | print "delete old backup '$d->[0]'\n"; | |
62 | unlink $d->[0]; | |
63 | } | |
64 | } | |
65 | ||
66 | __PACKAGE__->register_method ({ | |
c53b111f | 67 | name => 'keygen', |
13d44dc5 DM |
68 | path => 'keygen', |
69 | method => 'PUT', | |
70 | description => "Generate new cryptographic key for corosync.", | |
71 | parameters => { | |
72 | additionalProperties => 0, | |
73 | properties => { | |
74 | filename => { | |
75 | type => 'string', | |
76 | description => "Output file name" | |
77 | } | |
78 | }, | |
79 | }, | |
80 | returns => { type => 'null' }, | |
c53b111f | 81 | |
13d44dc5 DM |
82 | code => sub { |
83 | my ($param) = @_; | |
84 | ||
85 | my $filename = $param->{filename}; | |
86 | ||
87 | # test EUID | |
88 | $> == 0 || die "Error: Authorization key must be generated as root user.\n"; | |
89 | my $dirname = dirname($filename); | |
90 | my $basename = basename($filename); | |
91 | ||
92 | die "key file '$filename' already exists\n" if -e $filename; | |
93 | ||
94 | File::Path::make_path($dirname) if $dirname; | |
95 | ||
96 | my $cmd = ['corosync-keygen', '-l', '-k', $filename]; | |
c53b111f | 97 | PVE::Tools::run_command($cmd); |
13d44dc5 DM |
98 | |
99 | return undef; | |
100 | }}); | |
101 | ||
102 | __PACKAGE__->register_method ({ | |
c53b111f | 103 | name => 'create', |
13d44dc5 DM |
104 | path => 'create', |
105 | method => 'PUT', | |
106 | description => "Generate new cluster configuration.", | |
107 | parameters => { | |
108 | additionalProperties => 0, | |
109 | properties => { | |
110 | clustername => { | |
111 | description => "The name of the cluster.", | |
112 | type => 'string', format => 'pve-node', | |
113 | maxLength => 15, | |
114 | }, | |
115 | nodeid => { | |
116 | type => 'integer', | |
117 | description => "Node id for this node.", | |
118 | minimum => 1, | |
119 | optional => 1, | |
120 | }, | |
121 | votes => { | |
122 | type => 'integer', | |
123 | description => "Number of votes for this node.", | |
124 | minimum => 1, | |
125 | optional => 1, | |
126 | }, | |
14d0000a TL |
127 | bindnet0_addr => { |
128 | type => 'string', format => 'ip', | |
129 | description => "This specifies the network address the corosync ring 0". | |
130 | " executive should bind to and defaults to the local IP address of the node.", | |
131 | optional => 1, | |
132 | }, | |
133 | ring0_addr => { | |
134 | type => 'string', format => 'address', | |
135 | description => "Hostname (or IP) of the corosync ring0 address of this node.". | |
136 | " Defaults to the hostname of the node.", | |
137 | optional => 1, | |
138 | }, | |
14d0000a TL |
139 | bindnet1_addr => { |
140 | type => 'string', format => 'ip', | |
141 | description => "This specifies the network address the corosync ring 1". | |
142 | " executive should bind to and is optional.", | |
143 | optional => 1, | |
144 | }, | |
145 | ring1_addr => { | |
146 | type => 'string', format => 'address', | |
147 | description => "Hostname (or IP) of the corosync ring1 address, this". | |
148 | " needs an valid bindnet1_addr.", | |
149 | optional => 1, | |
150 | }, | |
151 | }, | |
13d44dc5 DM |
152 | }, |
153 | returns => { type => 'null' }, | |
c53b111f | 154 | |
13d44dc5 DM |
155 | code => sub { |
156 | my ($param) = @_; | |
157 | ||
158 | -f $clusterconf && die "cluster config '$clusterconf' already exists\n"; | |
159 | ||
6c0e95b3 | 160 | PVE::Cluster::setup_sshd_config(1); |
13d44dc5 DM |
161 | PVE::Cluster::setup_rootsshconfig(); |
162 | PVE::Cluster::setup_ssh_keys(); | |
163 | ||
164 | -f $authfile || __PACKAGE__->keygen({filename => $authfile}); | |
165 | ||
166 | -f $authfile || die "no authentication key available\n"; | |
167 | ||
168 | my $clustername = $param->{clustername}; | |
169 | ||
170 | $param->{nodeid} = 1 if !$param->{nodeid}; | |
171 | ||
172 | $param->{votes} = 1 if !defined($param->{votes}); | |
173 | ||
174 | my $nodename = PVE::INotify::nodename(); | |
c53b111f | 175 | |
13d44dc5 DM |
176 | my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); |
177 | ||
14d0000a TL |
178 | $param->{bindnet0_addr} = $local_ip_address |
179 | if !defined($param->{bindnet0_addr}); | |
180 | ||
181 | $param->{ring0_addr} = $nodename if !defined($param->{ring0_addr}); | |
182 | ||
183 | die "Param bindnet1_addr and ring1_addr are dependend, use both or none!\n" | |
184 | if (defined($param->{bindnet1_addr}) != defined($param->{ring1_addr})); | |
185 | ||
186 | my $bind_is_ipv6 = Net::IP::ip_is_ipv6($param->{bindnet0_addr}); | |
187 | ||
188 | # use string as here-doc format distracts more | |
189 | my $interfaces = "interface {\n ringnumber: 0\n" . | |
190 | " bindnetaddr: $param->{bindnet0_addr}\n }"; | |
191 | ||
192 | my $ring_addresses = "ring0_addr: $param->{ring0_addr}" ; | |
193 | ||
194 | # allow use of multiple rings (rrp) at cluster creation time | |
195 | if ($param->{bindnet1_addr}) { | |
196 | die "IPv6 and IPv4 cannot be mixed, use one or the other!\n" | |
197 | if Net::IP::ip_is_ipv6($param->{bindnet1_addr}) != $bind_is_ipv6; | |
198 | ||
14d0000a TL |
199 | $interfaces .= "\n interface {\n ringnumber: 1\n" . |
200 | " bindnetaddr: $param->{bindnet1_addr}\n }\n"; | |
201 | ||
606a8904 TL |
202 | $interfaces .= "rrp_mode: passive\n"; # only passive is stable and tested |
203 | ||
14d0000a TL |
204 | $ring_addresses .= "\n ring1_addr: $param->{ring1_addr}"; |
205 | ||
206 | } elsif($param->{rrp_mode} && $param->{rrp_mode} ne 'none') { | |
207 | ||
208 | warn "rrp_mode '$param->{rrp_mode}' useless when using only one". | |
209 | " ring, using 'none' instead"; | |
210 | # corosync defaults to none if only one interface is configured | |
211 | $param->{rrp_mode} = undef; | |
212 | ||
213 | } | |
214 | ||
13d44dc5 | 215 | # No, corosync cannot deduce this on its own |
14d0000a | 216 | my $ipversion = $bind_is_ipv6 ? 'ipv6' : 'ipv4'; |
13d44dc5 DM |
217 | |
218 | my $config = <<_EOD; | |
219 | totem { | |
220 | version: 2 | |
221 | secauth: on | |
222 | cluster_name: $clustername | |
223 | config_version: 1 | |
224 | ip_version: $ipversion | |
14d0000a | 225 | $interfaces |
13d44dc5 DM |
226 | } |
227 | ||
228 | nodelist { | |
229 | node { | |
14d0000a TL |
230 | $ring_addresses |
231 | name: $nodename | |
13d44dc5 DM |
232 | nodeid: $param->{nodeid} |
233 | quorum_votes: $param->{votes} | |
234 | } | |
235 | } | |
14d0000a | 236 | |
13d44dc5 DM |
237 | quorum { |
238 | provider: corosync_votequorum | |
239 | } | |
240 | ||
241 | logging { | |
242 | to_syslog: yes | |
243 | debug: off | |
244 | } | |
245 | _EOD | |
14d0000a | 246 | ; |
13d44dc5 DM |
247 | PVE::Tools::file_set_contents($clusterconf, $config); |
248 | ||
249 | PVE::Cluster::ssh_merge_keys(); | |
250 | ||
251 | PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address); | |
252 | ||
253 | PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1); | |
254 | ||
255 | PVE::Tools::run_command('systemctl restart pve-cluster'); # restart | |
256 | ||
257 | PVE::Tools::run_command('systemctl restart corosync'); # restart | |
c53b111f | 258 | |
13d44dc5 DM |
259 | return undef; |
260 | }}); | |
261 | ||
262 | __PACKAGE__->register_method ({ | |
c53b111f | 263 | name => 'addnode', |
13d44dc5 DM |
264 | path => 'addnode', |
265 | method => 'PUT', | |
266 | description => "Adds a node to the cluster configuration.", | |
267 | parameters => { | |
268 | additionalProperties => 0, | |
269 | properties => { | |
270 | node => PVE::JSONSchema::get_standard_option('pve-node'), | |
271 | nodeid => { | |
272 | type => 'integer', | |
273 | description => "Node id for this node.", | |
274 | minimum => 1, | |
275 | optional => 1, | |
276 | }, | |
277 | votes => { | |
278 | type => 'integer', | |
279 | description => "Number of votes for this node", | |
280 | minimum => 0, | |
281 | optional => 1, | |
282 | }, | |
283 | force => { | |
284 | type => 'boolean', | |
285 | description => "Do not throw error if node already exists.", | |
286 | optional => 1, | |
287 | }, | |
14d0000a TL |
288 | ring0_addr => { |
289 | type => 'string', format => 'address', | |
290 | description => "Hostname (or IP) of the corosync ring0 address of this node.". | |
291 | " Defaults to nodes hostname.", | |
292 | optional => 1, | |
293 | }, | |
294 | ring1_addr => { | |
295 | type => 'string', format => 'address', | |
296 | description => "Hostname (or IP) of the corosync ring1 address, this". | |
297 | " needs an valid bindnet1_addr.", | |
298 | optional => 1, | |
299 | }, | |
13d44dc5 DM |
300 | }, |
301 | }, | |
302 | returns => { type => 'null' }, | |
c53b111f | 303 | |
13d44dc5 DM |
304 | code => sub { |
305 | my ($param) = @_; | |
306 | ||
e73b85c0 TL |
307 | if (!$param->{force} && (-t STDIN || -t STDOUT)) { |
308 | die "error: `addnode` should not get called interactively!\nUse ". | |
309 | "`pvecm add <cluster-node>` to add a node to a cluster!\n"; | |
310 | } | |
311 | ||
13d44dc5 DM |
312 | PVE::Cluster::check_cfs_quorum(); |
313 | ||
b778c013 TL |
314 | my $code = sub { |
315 | my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); | |
316 | my $nodelist = PVE::Corosync::nodelist($conf); | |
317 | my $totem_cfg = PVE::Corosync::totem_config($conf); | |
318 | ||
319 | my $name = $param->{node}; | |
320 | ||
321 | # ensure we do not reuse an address, that can crash the whole cluster! | |
322 | my $check_duplicate_addr = sub { | |
323 | my $addr = shift; | |
324 | return if !defined($addr); | |
325 | ||
326 | while (my ($k, $v) = each %$nodelist) { | |
327 | next if $k eq $name; # allows re-adding a node if force is set | |
328 | if ($v->{ring0_addr} eq $addr || ($v->{ring1_addr} && $v->{ring1_addr} eq $addr)) { | |
329 | die "corosync: address '$addr' already defined by node '$k'\n"; | |
330 | } | |
77a1c918 | 331 | } |
b778c013 | 332 | }; |
77a1c918 | 333 | |
b778c013 TL |
334 | &$check_duplicate_addr($param->{ring0_addr}); |
335 | &$check_duplicate_addr($param->{ring1_addr}); | |
77a1c918 | 336 | |
b778c013 | 337 | $param->{ring0_addr} = $name if !$param->{ring0_addr}; |
14d0000a | 338 | |
b778c013 TL |
339 | die "corosync: using 'ring1_addr' parameter needs a configured ring 1 interface!\n" |
340 | if $param->{ring1_addr} && !defined($totem_cfg->{interface}->{1}); | |
14d0000a | 341 | |
b778c013 TL |
342 | die "corosync: ring 1 interface configured but 'ring1_addr' parameter not defined!\n" |
343 | if defined($totem_cfg->{interface}->{1}) && !defined($param->{ring1_addr}); | |
bb8583d0 | 344 | |
b778c013 TL |
345 | if (defined(my $res = $nodelist->{$name})) { |
346 | $param->{nodeid} = $res->{nodeid} if !$param->{nodeid}; | |
347 | $param->{votes} = $res->{quorum_votes} if !defined($param->{votes}); | |
13d44dc5 | 348 | |
b778c013 TL |
349 | if ($res->{quorum_votes} == $param->{votes} && |
350 | $res->{nodeid} == $param->{nodeid}) { | |
351 | print "node $name already defined\n"; | |
352 | if ($param->{force}) { | |
353 | exit (0); | |
354 | } else { | |
355 | exit (-1); | |
356 | } | |
13d44dc5 | 357 | } else { |
b778c013 | 358 | die "can't add existing node\n"; |
13d44dc5 | 359 | } |
b778c013 TL |
360 | } elsif (!$param->{nodeid}) { |
361 | my $nodeid = 1; | |
362 | ||
363 | while(1) { | |
364 | my $found = 0; | |
365 | foreach my $v (values %$nodelist) { | |
366 | if ($v->{nodeid} eq $nodeid) { | |
367 | $found = 1; | |
368 | $nodeid++; | |
369 | last; | |
370 | } | |
13d44dc5 | 371 | } |
b778c013 TL |
372 | last if !$found; |
373 | }; | |
13d44dc5 | 374 | |
b778c013 TL |
375 | $param->{nodeid} = $nodeid; |
376 | } | |
13d44dc5 | 377 | |
b778c013 | 378 | $param->{votes} = 1 if !defined($param->{votes}); |
13d44dc5 | 379 | |
b778c013 | 380 | PVE::Cluster::gen_local_dirs($name); |
13d44dc5 | 381 | |
b778c013 TL |
382 | eval { PVE::Cluster::ssh_merge_keys(); }; |
383 | warn $@ if $@; | |
13d44dc5 | 384 | |
b778c013 TL |
385 | $nodelist->{$name} = { |
386 | ring0_addr => $param->{ring0_addr}, | |
387 | nodeid => $param->{nodeid}, | |
388 | name => $name, | |
389 | }; | |
390 | $nodelist->{$name}->{ring1_addr} = $param->{ring1_addr} if $param->{ring1_addr}; | |
391 | $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes}; | |
392 | ||
393 | PVE::Corosync::update_nodelist($conf, $nodelist); | |
14d0000a | 394 | }; |
c53b111f | 395 | |
b778c013 TL |
396 | PVE::Cluster::cfs_lock_file('corosync.conf', 10, &$code); |
397 | die $@ if $@; | |
c53b111f | 398 | |
13d44dc5 DM |
399 | exit (0); |
400 | }}); | |
401 | ||
402 | ||
403 | __PACKAGE__->register_method ({ | |
c53b111f | 404 | name => 'delnode', |
13d44dc5 DM |
405 | path => 'delnode', |
406 | method => 'PUT', | |
407 | description => "Removes a node to the cluster configuration.", | |
408 | parameters => { | |
409 | additionalProperties => 0, | |
410 | properties => { | |
7aed8248 WL |
411 | node => { |
412 | type => 'string', | |
413 | description => "Hostname or IP of the corosync ring0 address of this node.", | |
414 | }, | |
13d44dc5 DM |
415 | }, |
416 | }, | |
417 | returns => { type => 'null' }, | |
c53b111f | 418 | |
13d44dc5 DM |
419 | code => sub { |
420 | my ($param) = @_; | |
421 | ||
422 | PVE::Cluster::check_cfs_quorum(); | |
423 | ||
b778c013 TL |
424 | my $code = sub { |
425 | my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); | |
426 | my $nodelist = PVE::Corosync::nodelist($conf); | |
427 | ||
428 | my $node; | |
429 | my $nodeid; | |
430 | ||
431 | foreach my $tmp_node (keys %$nodelist) { | |
432 | my $d = $nodelist->{$tmp_node}; | |
433 | my $ring0_addr = $d->{ring0_addr}; | |
434 | my $ring1_addr = $d->{ring1_addr}; | |
435 | if (($tmp_node eq $param->{node}) || | |
436 | (defined($ring0_addr) && ($ring0_addr eq $param->{node})) || | |
437 | (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) { | |
438 | $node = $tmp_node; | |
439 | $nodeid = $d->{nodeid}; | |
440 | last; | |
441 | } | |
7aed8248 | 442 | } |
7aed8248 | 443 | |
b778c013 | 444 | die "Node/IP: $param->{node} is not a known host of the cluster.\n" |
7aed8248 WL |
445 | if !defined($node); |
446 | ||
b778c013 TL |
447 | my $our_nodename = PVE::INotify::nodename(); |
448 | die "Cannot delete myself from cluster!\n" if $node eq $our_nodename; | |
449 | ||
450 | delete $nodelist->{$node}; | |
a8dbb454 | 451 | |
b778c013 | 452 | PVE::Corosync::update_nodelist($conf, $nodelist); |
7aed8248 | 453 | |
b778c013 TL |
454 | PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid]) |
455 | if defined($nodeid); | |
456 | }; | |
13d44dc5 | 457 | |
b778c013 TL |
458 | PVE::Cluster::cfs_lock_file('corosync.conf', 10, &$code); |
459 | die $@ if $@; | |
d09373b4 | 460 | |
13d44dc5 DM |
461 | return undef; |
462 | }}); | |
463 | ||
464 | __PACKAGE__->register_method ({ | |
c53b111f | 465 | name => 'add', |
13d44dc5 DM |
466 | path => 'add', |
467 | method => 'PUT', | |
468 | description => "Adds the current node to an existing cluster.", | |
469 | parameters => { | |
470 | additionalProperties => 0, | |
471 | properties => { | |
472 | hostname => { | |
473 | type => 'string', | |
474 | description => "Hostname (or IP) of an existing cluster member." | |
475 | }, | |
476 | nodeid => { | |
477 | type => 'integer', | |
478 | description => "Node id for this node.", | |
479 | minimum => 1, | |
480 | optional => 1, | |
481 | }, | |
482 | votes => { | |
483 | type => 'integer', | |
484 | description => "Number of votes for this node", | |
485 | minimum => 0, | |
486 | optional => 1, | |
487 | }, | |
488 | force => { | |
489 | type => 'boolean', | |
490 | description => "Do not throw error if node already exists.", | |
491 | optional => 1, | |
492 | }, | |
14d0000a TL |
493 | ring0_addr => { |
494 | type => 'string', format => 'address', | |
495 | description => "Hostname (or IP) of the corosync ring0 address of this node.". | |
496 | " Defaults to nodes hostname.", | |
497 | optional => 1, | |
498 | }, | |
499 | ring1_addr => { | |
500 | type => 'string', format => 'address', | |
501 | description => "Hostname (or IP) of the corosync ring1 address, this". | |
502 | " needs an valid configured ring 1 interface in the cluster.", | |
503 | optional => 1, | |
504 | }, | |
13d44dc5 DM |
505 | }, |
506 | }, | |
507 | returns => { type => 'null' }, | |
c53b111f | 508 | |
13d44dc5 DM |
509 | code => sub { |
510 | my ($param) = @_; | |
511 | ||
512 | my $nodename = PVE::INotify::nodename(); | |
513 | ||
6c0e95b3 | 514 | PVE::Cluster::setup_sshd_config(1); |
13d44dc5 DM |
515 | PVE::Cluster::setup_rootsshconfig(); |
516 | PVE::Cluster::setup_ssh_keys(); | |
517 | ||
518 | my $host = $param->{hostname}; | |
519 | ||
5a630d8f TL |
520 | my ($errors, $warnings) = ('', ''); |
521 | ||
522 | my $error = sub { | |
523 | my ($msg, $suppress) = @_; | |
524 | ||
525 | if ($suppress) { | |
526 | $warnings .= "* $msg\n"; | |
527 | } else { | |
528 | $errors .= "* $msg\n"; | |
529 | } | |
530 | }; | |
531 | ||
13d44dc5 | 532 | if (!$param->{force}) { |
c53b111f | 533 | |
13d44dc5 | 534 | if (-f $authfile) { |
5a630d8f | 535 | &$error("authentication key '$authfile' already exists", $param->{force}); |
13d44dc5 DM |
536 | } |
537 | ||
538 | if (-f $clusterconf) { | |
5a630d8f | 539 | &$error("cluster config '$clusterconf' already exists", $param->{force}); |
13d44dc5 DM |
540 | } |
541 | ||
542 | my $vmlist = PVE::Cluster::get_vmlist(); | |
543 | if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) { | |
5a630d8f | 544 | &$error("this host already contains virtual guests", $param->{force}); |
13d44dc5 DM |
545 | } |
546 | ||
cecd0323 | 547 | if (system("corosync-quorumtool -l >/dev/null 2>&1") == 0) { |
5a630d8f | 548 | &$error("corosync is already running, is this node already in a cluster?!", $param->{force}); |
13d44dc5 DM |
549 | } |
550 | } | |
551 | ||
f566b424 TL |
552 | # check if corosync ring IPs are configured on the current nodes interfaces |
553 | my $check_ip = sub { | |
554 | my $ip = shift; | |
555 | if (defined($ip)) { | |
1457a2a3 TL |
556 | if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) { |
557 | my $host = $ip; | |
558 | eval { $ip = PVE::Network::get_ip_from_hostname($host); }; | |
559 | if ($@) { | |
560 | &$error("cannot use '$host': $@\n") ; | |
561 | return; | |
562 | } | |
563 | } | |
564 | ||
f566b424 TL |
565 | my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32"; |
566 | my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr); | |
567 | ||
568 | &$error("cannot use IP '$ip', it must be configured exactly once on local node!\n") | |
569 | if (scalar(@$configured_ips) != 1); | |
570 | } | |
571 | }; | |
572 | ||
573 | &$check_ip($param->{ring0_addr}); | |
574 | &$check_ip($param->{ring1_addr}); | |
575 | ||
5a630d8f TL |
576 | warn "warning, ignore the following errors:\n$warnings" if $warnings; |
577 | die "detected the following error(s):\n$errors" if $errors; | |
578 | ||
13d44dc5 DM |
579 | # make sure known_hosts is on local filesystem |
580 | PVE::Cluster::ssh_unmerge_known_hosts(); | |
581 | ||
de4b4155 FG |
582 | my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"]; |
583 | PVE::Tools::run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {}, | |
584 | 'errmsg' => "unable to copy ssh ID"); | |
13d44dc5 DM |
585 | |
586 | $cmd = ['ssh', $host, '-o', 'BatchMode=yes', | |
587 | 'pvecm', 'addnode', $nodename, '--force', 1]; | |
588 | ||
589 | push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid}; | |
590 | ||
591 | push @$cmd, '--votes', $param->{votes} if defined($param->{votes}); | |
592 | ||
14d0000a TL |
593 | push @$cmd, '--ring0_addr', $param->{ring0_addr} if defined($param->{ring0_addr}); |
594 | ||
595 | push @$cmd, '--ring1_addr', $param->{ring1_addr} if defined($param->{ring1_addr}); | |
596 | ||
13d44dc5 DM |
597 | if (system (@$cmd) != 0) { |
598 | my $cmdtxt = join (' ', @$cmd); | |
599 | die "unable to add node: command failed ($cmdtxt)\n"; | |
600 | } | |
601 | ||
602 | my $tmpdir = "$libdir/.pvecm_add.tmp.$$"; | |
603 | mkdir $tmpdir; | |
604 | ||
605 | eval { | |
606 | print "copy corosync auth key\n"; | |
c53b111f | 607 | $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq', |
13d44dc5 DM |
608 | "[$host]:$authfile $clusterconf", $tmpdir]; |
609 | ||
610 | system(@$cmd) == 0 || die "can't rsync data from host '$host'\n"; | |
611 | ||
612 | mkdir "/etc/corosync"; | |
613 | my $confbase = basename($clusterconf); | |
614 | ||
615 | $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'"; | |
616 | system($cmd) == 0 || die "can't copy cluster configuration\n"; | |
617 | ||
618 | my $keybase = basename($authfile); | |
619 | system ("cp '$tmpdir/$keybase' '$authfile'") == 0 || | |
620 | die "can't copy '$tmpdir/$keybase' to '$authfile'\n"; | |
621 | ||
622 | print "stopping pve-cluster service\n"; | |
623 | ||
624 | system("umount $basedir -f >/dev/null 2>&1"); | |
625 | system("systemctl stop pve-cluster") == 0 || | |
626 | die "can't stop pve-cluster service\n"; | |
627 | ||
628 | backup_database(); | |
629 | ||
630 | unlink $dbfile; | |
631 | ||
632 | system("systemctl start pve-cluster") == 0 || | |
633 | die "starting pve-cluster failed\n"; | |
634 | ||
635 | system("systemctl start corosync"); | |
636 | ||
637 | # wait for quorum | |
638 | my $printqmsg = 1; | |
639 | while (!PVE::Cluster::check_cfs_quorum(1)) { | |
640 | if ($printqmsg) { | |
641 | print "waiting for quorum..."; | |
642 | STDOUT->flush(); | |
643 | $printqmsg = 0; | |
644 | } | |
645 | sleep(1); | |
646 | } | |
647 | print "OK\n" if !$printqmsg; | |
648 | ||
13d44dc5 DM |
649 | my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); |
650 | ||
651 | print "generating node certificates\n"; | |
c53b111f | 652 | PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address); |
13d44dc5 DM |
653 | |
654 | print "merge known_hosts file\n"; | |
655 | PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1); | |
656 | ||
657 | print "restart services\n"; | |
658 | # restart pvedaemon (changed certs) | |
659 | system("systemctl restart pvedaemon"); | |
660 | # restart pveproxy (changed certs) | |
661 | system("systemctl restart pveproxy"); | |
662 | ||
663 | print "successfully added node '$nodename' to cluster.\n"; | |
664 | }; | |
665 | my $err = $@; | |
666 | ||
667 | rmtree $tmpdir; | |
668 | ||
669 | die $err if $err; | |
670 | ||
671 | return undef; | |
672 | }}); | |
673 | ||
674 | __PACKAGE__->register_method ({ | |
c53b111f | 675 | name => 'status', |
13d44dc5 DM |
676 | path => 'status', |
677 | method => 'GET', | |
678 | description => "Displays the local view of the cluster status.", | |
679 | parameters => { | |
680 | additionalProperties => 0, | |
681 | properties => {}, | |
682 | }, | |
683 | returns => { type => 'null' }, | |
c53b111f | 684 | |
13d44dc5 DM |
685 | code => sub { |
686 | my ($param) = @_; | |
687 | ||
b6973a89 | 688 | PVE::Corosync::check_conf_exists(); |
eb51b829 | 689 | |
13d44dc5 DM |
690 | my $cmd = ['corosync-quorumtool', '-siH']; |
691 | ||
692 | exec (@$cmd); | |
693 | ||
694 | exit (-1); # should not be reached | |
695 | }}); | |
696 | ||
697 | __PACKAGE__->register_method ({ | |
c53b111f | 698 | name => 'nodes', |
13d44dc5 DM |
699 | path => 'nodes', |
700 | method => 'GET', | |
701 | description => "Displays the local view of the cluster nodes.", | |
702 | parameters => { | |
703 | additionalProperties => 0, | |
704 | properties => {}, | |
705 | }, | |
706 | returns => { type => 'null' }, | |
c53b111f | 707 | |
13d44dc5 DM |
708 | code => sub { |
709 | my ($param) = @_; | |
710 | ||
b6973a89 | 711 | PVE::Corosync::check_conf_exists(); |
eb51b829 | 712 | |
13d44dc5 DM |
713 | my $cmd = ['corosync-quorumtool', '-l']; |
714 | ||
715 | exec (@$cmd); | |
716 | ||
717 | exit (-1); # should not be reached | |
718 | }}); | |
719 | ||
720 | __PACKAGE__->register_method ({ | |
c53b111f | 721 | name => 'expected', |
13d44dc5 DM |
722 | path => 'expected', |
723 | method => 'PUT', | |
724 | description => "Tells corosync a new value of expected votes.", | |
725 | parameters => { | |
726 | additionalProperties => 0, | |
727 | properties => { | |
728 | expected => { | |
729 | type => 'integer', | |
730 | description => "Expected votes", | |
731 | minimum => 1, | |
732 | }, | |
733 | }, | |
734 | }, | |
735 | returns => { type => 'null' }, | |
c53b111f | 736 | |
13d44dc5 DM |
737 | code => sub { |
738 | my ($param) = @_; | |
739 | ||
b6973a89 | 740 | PVE::Corosync::check_conf_exists(); |
eb51b829 | 741 | |
13d44dc5 DM |
742 | my $cmd = ['corosync-quorumtool', '-e', $param->{expected}]; |
743 | ||
744 | exec (@$cmd); | |
745 | ||
746 | exit (-1); # should not be reached | |
747 | ||
748 | }}); | |
749 | ||
13d44dc5 | 750 | __PACKAGE__->register_method ({ |
c53b111f | 751 | name => 'updatecerts', |
13d44dc5 DM |
752 | path => 'updatecerts', |
753 | method => 'PUT', | |
754 | description => "Update node certificates (and generate all needed files/directories).", | |
755 | parameters => { | |
756 | additionalProperties => 0, | |
757 | properties => { | |
758 | force => { | |
759 | description => "Force generation of new SSL certifate.", | |
760 | type => 'boolean', | |
761 | optional => 1, | |
762 | }, | |
763 | silent => { | |
764 | description => "Ignore errors (i.e. when cluster has no quorum).", | |
765 | type => 'boolean', | |
766 | optional => 1, | |
767 | }, | |
768 | }, | |
769 | }, | |
770 | returns => { type => 'null' }, | |
771 | code => sub { | |
772 | my ($param) = @_; | |
773 | ||
6c0e95b3 | 774 | PVE::Cluster::setup_sshd_config(0); |
13d44dc5 DM |
775 | PVE::Cluster::setup_rootsshconfig(); |
776 | ||
777 | PVE::Cluster::gen_pve_vzdump_symlink(); | |
778 | ||
779 | if (!PVE::Cluster::check_cfs_quorum(1)) { | |
780 | return undef if $param->{silent}; | |
781 | die "no quorum - unable to update files\n"; | |
782 | } | |
783 | ||
784 | PVE::Cluster::setup_ssh_keys(); | |
785 | ||
786 | my $nodename = PVE::INotify::nodename(); | |
787 | ||
788 | my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); | |
789 | ||
790 | PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force}); | |
791 | PVE::Cluster::ssh_merge_keys(); | |
792 | PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address); | |
793 | PVE::Cluster::gen_pve_vzdump_files(); | |
794 | ||
795 | return undef; | |
796 | }}); | |
797 | ||
3a966e22 TL |
798 | __PACKAGE__->register_method ({ |
799 | name => 'mtunnel', | |
800 | path => 'mtunnel', | |
801 | method => 'POST', | |
802 | description => "Used by VM/CT migration - do not use manually.", | |
803 | parameters => { | |
804 | additionalProperties => 0, | |
f83d8153 TL |
805 | properties => { |
806 | get_migration_ip => { | |
807 | type => 'boolean', | |
808 | default => 0, | |
809 | description => 'return the migration IP, if configured', | |
810 | optional => 1, | |
811 | }, | |
812 | migration_network => { | |
813 | type => 'string', | |
814 | format => 'CIDR', | |
815 | description => 'the migration network used to detect the local migration IP', | |
816 | optional => 1, | |
817 | }, | |
d56a1ff3 WB |
818 | 'run-command' => { |
819 | type => 'boolean', | |
820 | description => 'Run a command with a tcp socket as standard input.' | |
821 | .' The IP address and port are printed via this' | |
822 | ." command's stdandard output first, each on a separate line.", | |
823 | optional => 1, | |
824 | }, | |
825 | 'extra-args' => PVE::JSONSchema::get_standard_option('extra-args'), | |
f83d8153 | 826 | }, |
3a966e22 TL |
827 | }, |
828 | returns => { type => 'null'}, | |
829 | code => sub { | |
830 | my ($param) = @_; | |
831 | ||
832 | if (!PVE::Cluster::check_cfs_quorum(1)) { | |
833 | print "no quorum\n"; | |
834 | return undef; | |
835 | } | |
836 | ||
d56a1ff3 | 837 | my $network = $param->{migration_network}; |
f83d8153 | 838 | if ($param->{get_migration_ip}) { |
d56a1ff3 WB |
839 | die "cannot use --run-command with --get_migration_ip\n" |
840 | if $param->{'run-command'}; | |
43f4c5f6 | 841 | if (my $ip = PVE::Cluster::get_local_migration_ip($network)) { |
f83d8153 TL |
842 | print "ip: '$ip'\n"; |
843 | } else { | |
43f4c5f6 | 844 | print "no ip\n"; |
f83d8153 TL |
845 | } |
846 | # do not keep tunnel open when asked for migration ip | |
847 | return undef; | |
848 | } | |
849 | ||
d56a1ff3 WB |
850 | if ($param->{'run-command'}) { |
851 | my $cmd = $param->{'extra-args'}; | |
852 | die "missing command\n" | |
853 | if !$cmd || !scalar(@$cmd); | |
854 | ||
855 | # Get an ip address to listen on, and find a free migration port | |
856 | my ($ip, $family); | |
857 | if (defined($network)) { | |
858 | $ip = PVE::Cluster::get_local_migration_ip($network) | |
859 | or die "failed to get migration IP address to listen on\n"; | |
860 | $family = Net::IP::ip_is_ipv6($ip) ? AF_INET6 : AF_INET; | |
861 | } else { | |
862 | my $nodename = PVE::INotify::nodename(); | |
863 | ($ip, $family) = PVE::Network::get_ip_from_hostname($nodename, 0); | |
864 | } | |
865 | my $port = PVE::Tools::next_migrate_port($family, $ip); | |
866 | ||
867 | # Wait for a client | |
868 | my $socket = IO::Socket::IP->new( | |
869 | Listen => 1, | |
870 | ReuseAddr => 1, | |
871 | Family => $family, | |
872 | Proto => &Socket::IPPROTO_TCP, | |
873 | GetAddrInfoFlags => 0, | |
874 | LocalAddr => $ip, | |
875 | LocalPort => $port, | |
876 | ) or die "failed to open socket: $!\n"; | |
877 | print "$ip\n$port\n"; | |
878 | *STDOUT->flush(); | |
879 | alarm 0; | |
880 | local $SIG{ALRM} = sub { die "timed out waiting for client\n" }; | |
881 | alarm 30; | |
882 | my $client = $socket->accept; | |
883 | alarm 0; | |
884 | close($socket); | |
885 | ||
886 | # We want that the command talks over the TCP socket and takes | |
887 | # ownership of it, so that when it closes it the connection is | |
888 | # terminated, so we need to be able to close the socket. So we | |
889 | # can't really use PVE::Tools::run_command(). | |
890 | my $pid = fork(); | |
891 | die "fork failed: $!\n" if !defined($pid); | |
892 | if (!$pid) { | |
893 | POSIX::dup2(fileno($client), 0); | |
894 | POSIX::dup2(fileno($client), 1); | |
895 | close($client); | |
896 | exec {$cmd->[0]} @$cmd or do { | |
897 | warn "exec failed: $!\n"; | |
898 | POSIX::_exit(1); | |
899 | }; | |
900 | } | |
901 | close($client); | |
902 | if (waitpid($pid, 0) != $pid) { | |
903 | kill(9 => $pid); | |
904 | 1 while waitpid($pid, 0) != $pid; | |
905 | } | |
906 | if (my $sig = ($? & 127)) { | |
907 | die "got signal $sig\n"; | |
908 | } elsif (my $exitcode = ($? >> 8)) { | |
909 | die "exit code $exitcode\n"; | |
910 | } | |
911 | return undef; | |
912 | } | |
913 | ||
3a966e22 TL |
914 | print "tunnel online\n"; |
915 | *STDOUT->flush(); | |
916 | ||
917 | while (my $line = <>) { | |
918 | chomp $line; | |
919 | last if $line =~ m/^quit$/; | |
920 | } | |
921 | ||
922 | return undef; | |
923 | }}); | |
924 | ||
13d44dc5 DM |
925 | |
926 | our $cmddef = { | |
927 | keygen => [ __PACKAGE__, 'keygen', ['filename']], | |
928 | create => [ __PACKAGE__, 'create', ['clustername']], | |
929 | add => [ __PACKAGE__, 'add', ['hostname']], | |
930 | addnode => [ __PACKAGE__, 'addnode', ['node']], | |
931 | delnode => [ __PACKAGE__, 'delnode', ['node']], | |
932 | status => [ __PACKAGE__, 'status' ], | |
933 | nodes => [ __PACKAGE__, 'nodes' ], | |
934 | expected => [ __PACKAGE__, 'expected', ['expected']], | |
935 | updatecerts => [ __PACKAGE__, 'updatecerts', []], | |
d56a1ff3 | 936 | mtunnel => [ __PACKAGE__, 'mtunnel', ['extra-args']], |
13d44dc5 DM |
937 | }; |
938 | ||
939 | 1; |