]>
Commit | Line | Data |
---|---|---|
963c06bb DM |
1 | package PVE::API2::ClusterConfig; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
b6973a89 | 5 | |
963c06bb DM |
6 | use PVE::Tools; |
7 | use PVE::SafeSyslog; | |
8 | use PVE::RESTHandler; | |
9 | use PVE::RPCEnvironment; | |
10 | use PVE::JSONSchema qw(get_standard_option); | |
11 | use PVE::Cluster; | |
6ed49eb1 | 12 | use PVE::APIClient::LWP; |
b6973a89 | 13 | use PVE::Corosync; |
963c06bb DM |
14 | |
15 | use base qw(PVE::RESTHandler); | |
16 | ||
74e09a93 TL |
17 | my $clusterconf = "/etc/pve/corosync.conf"; |
18 | my $authfile = "/etc/corosync/authkey"; | |
b184a69d | 19 | my $local_cluster_lock = "/var/lock/pvecm.lock"; |
74e09a93 | 20 | |
10c6810e TL |
21 | my $ring0_desc = { |
22 | type => 'string', format => 'address', | |
23 | description => "Hostname (or IP) of the corosync ring0 address of this node.", | |
24 | default => "Hostname of the node", | |
25 | optional => 1, | |
26 | }; | |
27 | PVE::JSONSchema::register_standard_option("corosync-ring0-addr", $ring0_desc); | |
28 | ||
29 | my $ring1_desc = { | |
30 | type => 'string', format => 'address', | |
31 | description => "Hostname (or IP) of the corosync ring1 address of this node.". | |
32 | " Requires a valid configured ring 1 (bindnet1_addr) in the cluster.", | |
33 | optional => 1, | |
34 | }; | |
35 | PVE::JSONSchema::register_standard_option("corosync-ring1-addr", $ring1_desc); | |
36 | ||
37 | my $nodeid_desc = { | |
38 | type => 'integer', | |
39 | description => "Node id for this node.", | |
40 | minimum => 1, | |
41 | optional => 1, | |
42 | }; | |
43 | PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc); | |
44 | ||
963c06bb DM |
45 | __PACKAGE__->register_method({ |
46 | name => 'index', | |
47 | path => '', | |
48 | method => 'GET', | |
49 | description => "Directory index.", | |
fb7c665a EK |
50 | permissions => { |
51 | check => ['perm', '/', [ 'Sys.Audit' ]], | |
52 | }, | |
963c06bb DM |
53 | parameters => { |
54 | additionalProperties => 0, | |
55 | properties => {}, | |
56 | }, | |
57 | returns => { | |
58 | type => 'array', | |
59 | items => { | |
60 | type => "object", | |
61 | properties => {}, | |
62 | }, | |
63 | links => [ { rel => 'child', href => "{name}" } ], | |
64 | }, | |
65 | code => sub { | |
66 | my ($param) = @_; | |
67 | ||
68 | my $result = [ | |
69 | { name => 'nodes' }, | |
70 | { name => 'totem' }, | |
6ed49eb1 TL |
71 | { name => 'join' }, |
72 | ]; | |
963c06bb DM |
73 | |
74 | return $result; | |
75 | }}); | |
76 | ||
74e09a93 TL |
77 | __PACKAGE__->register_method ({ |
78 | name => 'create', | |
79 | path => '', | |
80 | method => 'POST', | |
81 | protected => 1, | |
82 | description => "Generate new cluster configuration.", | |
83 | parameters => { | |
84 | additionalProperties => 0, | |
85 | properties => { | |
86 | clustername => { | |
87 | description => "The name of the cluster.", | |
88 | type => 'string', format => 'pve-node', | |
89 | maxLength => 15, | |
90 | }, | |
10c6810e | 91 | nodeid => get_standard_option('corosync-nodeid'), |
74e09a93 TL |
92 | votes => { |
93 | type => 'integer', | |
94 | description => "Number of votes for this node.", | |
95 | minimum => 1, | |
96 | optional => 1, | |
97 | }, | |
98 | bindnet0_addr => { | |
99 | type => 'string', format => 'ip', | |
100 | description => "This specifies the network address the corosync ring 0". | |
101 | " executive should bind to and defaults to the local IP address of the node.", | |
102 | optional => 1, | |
103 | }, | |
10c6810e | 104 | ring0_addr => get_standard_option('corosync-ring0-addr'), |
74e09a93 TL |
105 | bindnet1_addr => { |
106 | type => 'string', format => 'ip', | |
107 | description => "This specifies the network address the corosync ring 1". | |
108 | " executive should bind to and is optional.", | |
109 | optional => 1, | |
110 | }, | |
10c6810e | 111 | ring1_addr => get_standard_option('corosync-ring1-addr'), |
74e09a93 TL |
112 | }, |
113 | }, | |
333a9bc8 | 114 | returns => { type => 'string' }, |
74e09a93 TL |
115 | code => sub { |
116 | my ($param) = @_; | |
117 | ||
118 | -f $clusterconf && die "cluster config '$clusterconf' already exists\n"; | |
119 | ||
333a9bc8 TL |
120 | my $rpcenv = PVE::RPCEnvironment::get(); |
121 | my $authuser = $rpcenv->get_user(); | |
122 | ||
b184a69d | 123 | my $code = sub { |
333a9bc8 TL |
124 | PVE::Cluster::setup_sshd_config(1); |
125 | PVE::Cluster::setup_rootsshconfig(); | |
126 | PVE::Cluster::setup_ssh_keys(); | |
74e09a93 | 127 | |
333a9bc8 TL |
128 | PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile]) |
129 | if !-f $authfile; | |
130 | die "no authentication key available\n" if -f !$authfile; | |
74e09a93 | 131 | |
333a9bc8 | 132 | my $nodename = PVE::INotify::nodename(); |
74e09a93 | 133 | |
333a9bc8 TL |
134 | # get the corosync basis config for the new cluster |
135 | my $config = PVE::Corosync::create_conf($nodename, %$param); | |
74e09a93 | 136 | |
333a9bc8 TL |
137 | print "Writing corosync config to /etc/pve/corosync.conf\n"; |
138 | PVE::Corosync::atomic_write_conf($config); | |
74e09a93 | 139 | |
333a9bc8 TL |
140 | my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); |
141 | PVE::Cluster::ssh_merge_keys(); | |
142 | PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address); | |
143 | PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1); | |
74e09a93 | 144 | |
333a9bc8 TL |
145 | print "Restart corosync and cluster filesystem\n"; |
146 | PVE::Tools::run_command('systemctl restart corosync pve-cluster'); | |
147 | }; | |
74e09a93 | 148 | |
b184a69d TL |
149 | my $worker = sub { |
150 | PVE::Tools::lock_file($local_cluster_lock, 10, $code); | |
151 | die $@ if $@; | |
152 | }; | |
153 | ||
65ee8001 | 154 | return $rpcenv->fork_worker('clustercreate', $param->{clustername}, $authuser, $worker); |
74e09a93 TL |
155 | }}); |
156 | ||
963c06bb DM |
157 | __PACKAGE__->register_method({ |
158 | name => 'nodes', | |
159 | path => 'nodes', | |
160 | method => 'GET', | |
161 | description => "Corosync node list.", | |
fb7c665a EK |
162 | permissions => { |
163 | check => ['perm', '/', [ 'Sys.Audit' ]], | |
164 | }, | |
963c06bb DM |
165 | parameters => { |
166 | additionalProperties => 0, | |
167 | properties => {}, | |
168 | }, | |
169 | returns => { | |
170 | type => 'array', | |
171 | items => { | |
172 | type => "object", | |
173 | properties => { | |
174 | node => { type => 'string' }, | |
175 | }, | |
176 | }, | |
177 | links => [ { rel => 'child', href => "{node}" } ], | |
178 | }, | |
179 | code => sub { | |
180 | my ($param) = @_; | |
181 | ||
182 | ||
183 | my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); | |
b6973a89 | 184 | my $nodelist = PVE::Corosync::nodelist($conf); |
963c06bb DM |
185 | |
186 | return PVE::RESTHandler::hash_to_array($nodelist, 'node'); | |
187 | }}); | |
188 | ||
1d26c202 TL |
189 | # lock method to ensure local and cluster wide atomicity |
190 | # if we're a single node cluster just lock locally, we have no other cluster | |
191 | # node which we could contend with, else also acquire a cluster wide lock | |
192 | my $config_change_lock = sub { | |
193 | my ($code) = @_; | |
194 | ||
b184a69d | 195 | PVE::Tools::lock_file($local_cluster_lock, 10, sub { |
1d26c202 TL |
196 | PVE::Cluster::cfs_update(1); |
197 | my $members = PVE::Cluster::get_members(); | |
198 | if (scalar(keys %$members) > 1) { | |
199 | return PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code); | |
200 | } else { | |
201 | return $code->(); | |
202 | } | |
203 | }); | |
204 | }; | |
205 | ||
1d26c202 TL |
206 | __PACKAGE__->register_method ({ |
207 | name => 'addnode', | |
208 | path => 'nodes/{node}', | |
209 | method => 'POST', | |
210 | protected => 1, | |
211 | description => "Adds a node to the cluster configuration.", | |
212 | parameters => { | |
213 | additionalProperties => 0, | |
214 | properties => { | |
215 | node => get_standard_option('pve-node'), | |
10c6810e | 216 | nodeid => get_standard_option('corosync-nodeid'), |
1d26c202 TL |
217 | votes => { |
218 | type => 'integer', | |
219 | description => "Number of votes for this node", | |
220 | minimum => 0, | |
221 | optional => 1, | |
222 | }, | |
223 | force => { | |
224 | type => 'boolean', | |
225 | description => "Do not throw error if node already exists.", | |
226 | optional => 1, | |
227 | }, | |
10c6810e TL |
228 | ring0_addr => get_standard_option('corosync-ring0-addr'), |
229 | ring1_addr => get_standard_option('corosync-ring1-addr'), | |
1d26c202 TL |
230 | }, |
231 | }, | |
331d957b TL |
232 | returns => { |
233 | type => "object", | |
234 | properties => { | |
235 | corosync_authkey => { | |
236 | type => 'string', | |
237 | }, | |
238 | corosync_conf => { | |
239 | type => 'string', | |
240 | } | |
241 | }, | |
242 | }, | |
1d26c202 TL |
243 | code => sub { |
244 | my ($param) = @_; | |
245 | ||
246 | PVE::Cluster::check_cfs_quorum(); | |
247 | ||
248 | my $code = sub { | |
249 | my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); | |
250 | my $nodelist = PVE::Corosync::nodelist($conf); | |
251 | my $totem_cfg = PVE::Corosync::totem_config($conf); | |
252 | ||
253 | my $name = $param->{node}; | |
254 | ||
255 | # ensure we do not reuse an address, that can crash the whole cluster! | |
256 | my $check_duplicate_addr = sub { | |
257 | my $addr = shift; | |
258 | return if !defined($addr); | |
259 | ||
260 | while (my ($k, $v) = each %$nodelist) { | |
261 | next if $k eq $name; # allows re-adding a node if force is set | |
262 | if ($v->{ring0_addr} eq $addr || ($v->{ring1_addr} && $v->{ring1_addr} eq $addr)) { | |
263 | die "corosync: address '$addr' already defined by node '$k'\n"; | |
264 | } | |
265 | } | |
266 | }; | |
267 | ||
268 | &$check_duplicate_addr($param->{ring0_addr}); | |
269 | &$check_duplicate_addr($param->{ring1_addr}); | |
270 | ||
271 | $param->{ring0_addr} = $name if !$param->{ring0_addr}; | |
272 | ||
273 | die "corosync: using 'ring1_addr' parameter needs a configured ring 1 interface!\n" | |
274 | if $param->{ring1_addr} && !defined($totem_cfg->{interface}->{1}); | |
275 | ||
276 | die "corosync: ring 1 interface configured but 'ring1_addr' parameter not defined!\n" | |
277 | if defined($totem_cfg->{interface}->{1}) && !defined($param->{ring1_addr}); | |
278 | ||
279 | if (defined(my $res = $nodelist->{$name})) { | |
280 | $param->{nodeid} = $res->{nodeid} if !$param->{nodeid}; | |
281 | $param->{votes} = $res->{quorum_votes} if !defined($param->{votes}); | |
282 | ||
283 | if ($res->{quorum_votes} == $param->{votes} && | |
284 | $res->{nodeid} == $param->{nodeid} && $param->{force}) { | |
285 | print "forcing overwrite of configured node '$name'\n"; | |
286 | } else { | |
287 | die "can't add existing node '$name'\n"; | |
288 | } | |
289 | } elsif (!$param->{nodeid}) { | |
290 | my $nodeid = 1; | |
291 | ||
292 | while(1) { | |
293 | my $found = 0; | |
294 | foreach my $v (values %$nodelist) { | |
295 | if ($v->{nodeid} eq $nodeid) { | |
296 | $found = 1; | |
297 | $nodeid++; | |
298 | last; | |
299 | } | |
300 | } | |
301 | last if !$found; | |
302 | }; | |
303 | ||
304 | $param->{nodeid} = $nodeid; | |
305 | } | |
306 | ||
307 | $param->{votes} = 1 if !defined($param->{votes}); | |
308 | ||
309 | PVE::Cluster::gen_local_dirs($name); | |
310 | ||
311 | eval { PVE::Cluster::ssh_merge_keys(); }; | |
312 | warn $@ if $@; | |
313 | ||
314 | $nodelist->{$name} = { | |
315 | ring0_addr => $param->{ring0_addr}, | |
316 | nodeid => $param->{nodeid}, | |
317 | name => $name, | |
318 | }; | |
319 | $nodelist->{$name}->{ring1_addr} = $param->{ring1_addr} if $param->{ring1_addr}; | |
320 | $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes}; | |
321 | ||
855671ec TL |
322 | PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster"); |
323 | ||
1d26c202 TL |
324 | PVE::Corosync::update_nodelist($conf, $nodelist); |
325 | }; | |
326 | ||
327 | $config_change_lock->($code); | |
328 | die $@ if $@; | |
329 | ||
331d957b TL |
330 | my $res = { |
331 | corosync_authkey => PVE::Tools::file_get_contents($authfile), | |
332 | corosync_conf => PVE::Tools::file_get_contents($clusterconf), | |
333 | }; | |
334 | ||
335 | return $res; | |
1d26c202 TL |
336 | }}); |
337 | ||
338 | ||
339 | __PACKAGE__->register_method ({ | |
340 | name => 'delnode', | |
341 | path => 'nodes/{node}', | |
342 | method => 'DELETE', | |
343 | protected => 1, | |
344 | description => "Removes a node from the cluster configuration.", | |
345 | parameters => { | |
346 | additionalProperties => 0, | |
347 | properties => { | |
348 | node => get_standard_option('pve-node'), | |
349 | }, | |
350 | }, | |
351 | returns => { type => 'null' }, | |
352 | code => sub { | |
353 | my ($param) = @_; | |
354 | ||
355 | my $local_node = PVE::INotify::nodename(); | |
356 | die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node; | |
357 | ||
358 | PVE::Cluster::check_cfs_quorum(); | |
359 | ||
360 | my $code = sub { | |
361 | my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); | |
362 | my $nodelist = PVE::Corosync::nodelist($conf); | |
363 | ||
364 | my $node; | |
365 | my $nodeid; | |
366 | ||
367 | foreach my $tmp_node (keys %$nodelist) { | |
368 | my $d = $nodelist->{$tmp_node}; | |
369 | my $ring0_addr = $d->{ring0_addr}; | |
370 | my $ring1_addr = $d->{ring1_addr}; | |
371 | if (($tmp_node eq $param->{node}) || | |
372 | (defined($ring0_addr) && ($ring0_addr eq $param->{node})) || | |
373 | (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) { | |
374 | $node = $tmp_node; | |
375 | $nodeid = $d->{nodeid}; | |
376 | last; | |
377 | } | |
378 | } | |
379 | ||
380 | die "Node/IP: $param->{node} is not a known host of the cluster.\n" | |
381 | if !defined($node); | |
382 | ||
855671ec TL |
383 | PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster"); |
384 | ||
1d26c202 TL |
385 | delete $nodelist->{$node}; |
386 | ||
387 | PVE::Corosync::update_nodelist($conf, $nodelist); | |
388 | ||
389 | PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid]) if defined($nodeid); | |
390 | }; | |
391 | ||
392 | $config_change_lock->($code); | |
393 | die $@ if $@; | |
394 | ||
395 | return undef; | |
396 | }}); | |
397 | ||
fca7797d TL |
398 | __PACKAGE__->register_method ({ |
399 | name => 'join_info', | |
400 | path => 'join', | |
401 | method => 'GET', | |
402 | description => "Get information needed to join this cluster over the connected node.", | |
403 | parameters => { | |
404 | additionalProperties => 0, | |
405 | properties => { | |
406 | node => get_standard_option('pve-node', { | |
407 | description => "The node for which the joinee gets the nodeinfo. ", | |
408 | default => "current connected node", | |
409 | optional => 1, | |
410 | }), | |
411 | }, | |
412 | }, | |
413 | returns => { | |
414 | type => 'object', | |
415 | additionalProperties => 0, | |
416 | properties => { | |
417 | nodelist => { | |
418 | type => 'array', | |
419 | items => { | |
420 | type => "object", | |
421 | additionalProperties => 1, | |
422 | properties => { | |
423 | name => get_standard_option('pve-node'), | |
424 | nodeid => get_standard_option('corosync-nodeid'), | |
425 | ring0_addr => get_standard_option('corosync-ring0-addr'), | |
426 | quorum_votes => { type => 'integer', minimum => 0 }, | |
427 | pve_addr => { type => 'string', format => 'ip' }, | |
428 | pve_fp => get_standard_option('fingerprint-sha256'), | |
429 | }, | |
430 | }, | |
431 | }, | |
432 | preferred_node => get_standard_option('pve-node'), | |
433 | totem => { type => 'object' }, | |
434 | config_digest => { type => 'string' }, | |
435 | }, | |
436 | }, | |
437 | code => sub { | |
438 | my ($param) = @_; | |
439 | ||
440 | my $nodename = $param->{node} // PVE::INotify::nodename(); | |
441 | ||
442 | PVE::Cluster::cfs_update(1); | |
443 | my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); | |
444 | ||
445 | die "node is not in a cluster, no join info available!\n" | |
446 | if !($conf && $conf->{main}); | |
447 | ||
448 | my $totem_cfg = $conf->{main}->{totem} // {}; | |
449 | my $nodelist = $conf->{main}->{nodelist}->{node} // {}; | |
450 | my $corosync_config_digest = $conf->{digest}; | |
451 | ||
452 | die "unknown node '$nodename'\n" if ! $nodelist->{$nodename}; | |
453 | ||
454 | foreach my $name (keys %$nodelist) { | |
455 | my $node = $nodelist->{$name}; | |
456 | $node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name); | |
457 | $node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name)); | |
458 | } | |
459 | ||
460 | my $res = { | |
461 | nodelist => [ values %$nodelist ], | |
462 | preferred_node => $nodename, | |
463 | totem => $totem_cfg, | |
464 | config_digest => $corosync_config_digest, | |
465 | }; | |
466 | ||
467 | return $res; | |
468 | }}); | |
469 | ||
6ed49eb1 TL |
470 | __PACKAGE__->register_method ({ |
471 | name => 'join', | |
472 | path => 'join', | |
473 | method => 'POST', | |
474 | protected => 1, | |
475 | description => "Joins this node into an existing cluster.", | |
476 | parameters => { | |
477 | additionalProperties => 0, | |
478 | properties => { | |
479 | hostname => { | |
480 | type => 'string', | |
481 | description => "Hostname (or IP) of an existing cluster member." | |
482 | }, | |
10c6810e | 483 | nodeid => get_standard_option('corosync-nodeid'), |
6ed49eb1 TL |
484 | votes => { |
485 | type => 'integer', | |
486 | description => "Number of votes for this node", | |
487 | minimum => 0, | |
488 | optional => 1, | |
489 | }, | |
490 | force => { | |
491 | type => 'boolean', | |
492 | description => "Do not throw error if node already exists.", | |
493 | optional => 1, | |
494 | }, | |
10c6810e | 495 | ring0_addr => get_standard_option('corosync-ring0-addr', { |
072ba2e0 | 496 | default => "IP resolved by node's hostname", |
10c6810e TL |
497 | }), |
498 | ring1_addr => get_standard_option('corosync-ring1-addr'), | |
6ed49eb1 TL |
499 | fingerprint => get_standard_option('fingerprint-sha256'), |
500 | password => { | |
501 | description => "Superuser (root) password of peer node.", | |
502 | type => 'string', | |
503 | maxLength => 128, | |
504 | }, | |
505 | }, | |
506 | }, | |
507 | returns => { type => 'string' }, | |
508 | code => sub { | |
509 | my ($param) = @_; | |
510 | ||
511 | my $rpcenv = PVE::RPCEnvironment::get(); | |
512 | my $authuser = $rpcenv->get_user(); | |
513 | ||
514 | my $worker = sub { | |
b184a69d TL |
515 | PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::join, $param); |
516 | die $@ if $@; | |
6ed49eb1 TL |
517 | }; |
518 | ||
65ee8001 | 519 | return $rpcenv->fork_worker('clusterjoin', $param->{hostname}, $authuser, $worker); |
6ed49eb1 TL |
520 | }}); |
521 | ||
522 | ||
963c06bb DM |
523 | __PACKAGE__->register_method({ |
524 | name => 'totem', | |
525 | path => 'totem', | |
526 | method => 'GET', | |
527 | description => "Get corosync totem protocol settings.", | |
fb7c665a EK |
528 | permissions => { |
529 | check => ['perm', '/', [ 'Sys.Audit' ]], | |
530 | }, | |
963c06bb DM |
531 | parameters => { |
532 | additionalProperties => 0, | |
533 | properties => {}, | |
534 | }, | |
535 | returns => { | |
536 | type => "object", | |
537 | properties => {}, | |
538 | }, | |
539 | code => sub { | |
540 | my ($param) = @_; | |
541 | ||
542 | ||
543 | my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); | |
544 | ||
496de919 TL |
545 | my $totem_cfg = $conf->{main}->{totem}; |
546 | ||
547 | return $totem_cfg; | |
963c06bb DM |
548 | }}); |
549 | ||
550 | 1; |