]>
Commit | Line | Data |
---|---|---|
963c06bb DM |
1 | package PVE::API2::ClusterConfig; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
b6973a89 | 5 | |
963c06bb DM |
6 | use PVE::Tools; |
7 | use PVE::SafeSyslog; | |
8 | use PVE::RESTHandler; | |
9 | use PVE::RPCEnvironment; | |
10 | use PVE::JSONSchema qw(get_standard_option); | |
11 | use PVE::Cluster; | |
6ed49eb1 | 12 | use PVE::APIClient::LWP; |
b6973a89 | 13 | use PVE::Corosync; |
963c06bb DM |
14 | |
15 | use base qw(PVE::RESTHandler); | |
16 | ||
74e09a93 TL |
17 | my $clusterconf = "/etc/pve/corosync.conf"; |
18 | my $authfile = "/etc/corosync/authkey"; | |
b184a69d | 19 | my $local_cluster_lock = "/var/lock/pvecm.lock"; |
74e09a93 | 20 | |
10c6810e TL |
21 | my $ring0_desc = { |
22 | type => 'string', format => 'address', | |
23 | description => "Hostname (or IP) of the corosync ring0 address of this node.", | |
24 | default => "Hostname of the node", | |
25 | optional => 1, | |
26 | }; | |
27 | PVE::JSONSchema::register_standard_option("corosync-ring0-addr", $ring0_desc); | |
28 | ||
29 | my $ring1_desc = { | |
30 | type => 'string', format => 'address', | |
31 | description => "Hostname (or IP) of the corosync ring1 address of this node.". | |
32 | " Requires a valid configured ring 1 (bindnet1_addr) in the cluster.", | |
33 | optional => 1, | |
34 | }; | |
35 | PVE::JSONSchema::register_standard_option("corosync-ring1-addr", $ring1_desc); | |
36 | ||
37 | my $nodeid_desc = { | |
38 | type => 'integer', | |
39 | description => "Node id for this node.", | |
40 | minimum => 1, | |
41 | optional => 1, | |
42 | }; | |
43 | PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc); | |
44 | ||
963c06bb DM |
45 | __PACKAGE__->register_method({ |
46 | name => 'index', | |
47 | path => '', | |
48 | method => 'GET', | |
49 | description => "Directory index.", | |
fb7c665a EK |
50 | permissions => { |
51 | check => ['perm', '/', [ 'Sys.Audit' ]], | |
52 | }, | |
963c06bb DM |
53 | parameters => { |
54 | additionalProperties => 0, | |
55 | properties => {}, | |
56 | }, | |
57 | returns => { | |
58 | type => 'array', | |
59 | items => { | |
60 | type => "object", | |
61 | properties => {}, | |
62 | }, | |
63 | links => [ { rel => 'child', href => "{name}" } ], | |
64 | }, | |
65 | code => sub { | |
66 | my ($param) = @_; | |
67 | ||
68 | my $result = [ | |
69 | { name => 'nodes' }, | |
70 | { name => 'totem' }, | |
6ed49eb1 TL |
71 | { name => 'join' }, |
72 | ]; | |
963c06bb DM |
73 | |
74 | return $result; | |
75 | }}); | |
76 | ||
74e09a93 TL |
77 | __PACKAGE__->register_method ({ |
78 | name => 'create', | |
79 | path => '', | |
80 | method => 'POST', | |
81 | protected => 1, | |
82 | description => "Generate new cluster configuration.", | |
83 | parameters => { | |
84 | additionalProperties => 0, | |
85 | properties => { | |
86 | clustername => { | |
87 | description => "The name of the cluster.", | |
88 | type => 'string', format => 'pve-node', | |
89 | maxLength => 15, | |
90 | }, | |
10c6810e | 91 | nodeid => get_standard_option('corosync-nodeid'), |
74e09a93 TL |
92 | votes => { |
93 | type => 'integer', | |
94 | description => "Number of votes for this node.", | |
95 | minimum => 1, | |
96 | optional => 1, | |
97 | }, | |
98 | bindnet0_addr => { | |
99 | type => 'string', format => 'ip', | |
100 | description => "This specifies the network address the corosync ring 0". | |
101 | " executive should bind to and defaults to the local IP address of the node.", | |
102 | optional => 1, | |
103 | }, | |
10c6810e | 104 | ring0_addr => get_standard_option('corosync-ring0-addr'), |
74e09a93 TL |
105 | bindnet1_addr => { |
106 | type => 'string', format => 'ip', | |
107 | description => "This specifies the network address the corosync ring 1". | |
108 | " executive should bind to and is optional.", | |
109 | optional => 1, | |
110 | }, | |
10c6810e | 111 | ring1_addr => get_standard_option('corosync-ring1-addr'), |
74e09a93 TL |
112 | }, |
113 | }, | |
333a9bc8 | 114 | returns => { type => 'string' }, |
74e09a93 TL |
115 | code => sub { |
116 | my ($param) = @_; | |
117 | ||
118 | -f $clusterconf && die "cluster config '$clusterconf' already exists\n"; | |
119 | ||
333a9bc8 TL |
120 | my $rpcenv = PVE::RPCEnvironment::get(); |
121 | my $authuser = $rpcenv->get_user(); | |
122 | ||
b184a69d | 123 | my $code = sub { |
17a4445f | 124 | STDOUT->autoflush(); |
333a9bc8 TL |
125 | PVE::Cluster::setup_sshd_config(1); |
126 | PVE::Cluster::setup_rootsshconfig(); | |
127 | PVE::Cluster::setup_ssh_keys(); | |
74e09a93 | 128 | |
333a9bc8 TL |
129 | PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile]) |
130 | if !-f $authfile; | |
131 | die "no authentication key available\n" if -f !$authfile; | |
74e09a93 | 132 | |
333a9bc8 | 133 | my $nodename = PVE::INotify::nodename(); |
74e09a93 | 134 | |
333a9bc8 TL |
135 | # get the corosync basis config for the new cluster |
136 | my $config = PVE::Corosync::create_conf($nodename, %$param); | |
74e09a93 | 137 | |
333a9bc8 TL |
138 | print "Writing corosync config to /etc/pve/corosync.conf\n"; |
139 | PVE::Corosync::atomic_write_conf($config); | |
74e09a93 | 140 | |
333a9bc8 TL |
141 | my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); |
142 | PVE::Cluster::ssh_merge_keys(); | |
143 | PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address); | |
144 | PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1); | |
74e09a93 | 145 | |
333a9bc8 TL |
146 | print "Restart corosync and cluster filesystem\n"; |
147 | PVE::Tools::run_command('systemctl restart corosync pve-cluster'); | |
148 | }; | |
74e09a93 | 149 | |
b184a69d TL |
150 | my $worker = sub { |
151 | PVE::Tools::lock_file($local_cluster_lock, 10, $code); | |
152 | die $@ if $@; | |
153 | }; | |
154 | ||
65ee8001 | 155 | return $rpcenv->fork_worker('clustercreate', $param->{clustername}, $authuser, $worker); |
74e09a93 TL |
156 | }}); |
157 | ||
963c06bb DM |
158 | __PACKAGE__->register_method({ |
159 | name => 'nodes', | |
160 | path => 'nodes', | |
161 | method => 'GET', | |
162 | description => "Corosync node list.", | |
fb7c665a EK |
163 | permissions => { |
164 | check => ['perm', '/', [ 'Sys.Audit' ]], | |
165 | }, | |
963c06bb DM |
166 | parameters => { |
167 | additionalProperties => 0, | |
168 | properties => {}, | |
169 | }, | |
170 | returns => { | |
171 | type => 'array', | |
172 | items => { | |
173 | type => "object", | |
174 | properties => { | |
175 | node => { type => 'string' }, | |
176 | }, | |
177 | }, | |
178 | links => [ { rel => 'child', href => "{node}" } ], | |
179 | }, | |
180 | code => sub { | |
181 | my ($param) = @_; | |
182 | ||
183 | ||
184 | my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); | |
b6973a89 | 185 | my $nodelist = PVE::Corosync::nodelist($conf); |
963c06bb DM |
186 | |
187 | return PVE::RESTHandler::hash_to_array($nodelist, 'node'); | |
188 | }}); | |
189 | ||
1d26c202 TL |
190 | # lock method to ensure local and cluster wide atomicity |
191 | # if we're a single node cluster just lock locally, we have no other cluster | |
192 | # node which we could contend with, else also acquire a cluster wide lock | |
193 | my $config_change_lock = sub { | |
194 | my ($code) = @_; | |
195 | ||
b184a69d | 196 | PVE::Tools::lock_file($local_cluster_lock, 10, sub { |
1d26c202 TL |
197 | PVE::Cluster::cfs_update(1); |
198 | my $members = PVE::Cluster::get_members(); | |
199 | if (scalar(keys %$members) > 1) { | |
0f147e2a | 200 | my $res = PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code); |
282ce02b TL |
201 | |
202 | # cfs_lock_file only sets $@ but lock_file doesn't propagates $@ unless we die here | |
0f147e2a | 203 | die $@ if defined($@); |
282ce02b | 204 | |
0f147e2a | 205 | return $res; |
1d26c202 TL |
206 | } else { |
207 | return $code->(); | |
208 | } | |
209 | }); | |
210 | }; | |
211 | ||
1d26c202 TL |
212 | __PACKAGE__->register_method ({ |
213 | name => 'addnode', | |
214 | path => 'nodes/{node}', | |
215 | method => 'POST', | |
216 | protected => 1, | |
79cf5a70 | 217 | description => "Adds a node to the cluster configuration. This call is for internal use.", |
1d26c202 TL |
218 | parameters => { |
219 | additionalProperties => 0, | |
220 | properties => { | |
221 | node => get_standard_option('pve-node'), | |
10c6810e | 222 | nodeid => get_standard_option('corosync-nodeid'), |
1d26c202 TL |
223 | votes => { |
224 | type => 'integer', | |
225 | description => "Number of votes for this node", | |
226 | minimum => 0, | |
227 | optional => 1, | |
228 | }, | |
229 | force => { | |
230 | type => 'boolean', | |
231 | description => "Do not throw error if node already exists.", | |
232 | optional => 1, | |
233 | }, | |
10c6810e TL |
234 | ring0_addr => get_standard_option('corosync-ring0-addr'), |
235 | ring1_addr => get_standard_option('corosync-ring1-addr'), | |
1d26c202 TL |
236 | }, |
237 | }, | |
331d957b TL |
238 | returns => { |
239 | type => "object", | |
240 | properties => { | |
241 | corosync_authkey => { | |
242 | type => 'string', | |
243 | }, | |
244 | corosync_conf => { | |
245 | type => 'string', | |
246 | } | |
247 | }, | |
248 | }, | |
1d26c202 TL |
249 | code => sub { |
250 | my ($param) = @_; | |
251 | ||
252 | PVE::Cluster::check_cfs_quorum(); | |
253 | ||
254 | my $code = sub { | |
255 | my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); | |
256 | my $nodelist = PVE::Corosync::nodelist($conf); | |
257 | my $totem_cfg = PVE::Corosync::totem_config($conf); | |
258 | ||
259 | my $name = $param->{node}; | |
260 | ||
261 | # ensure we do not reuse an address, that can crash the whole cluster! | |
262 | my $check_duplicate_addr = sub { | |
263 | my $addr = shift; | |
264 | return if !defined($addr); | |
265 | ||
266 | while (my ($k, $v) = each %$nodelist) { | |
267 | next if $k eq $name; # allows re-adding a node if force is set | |
dd92cac5 TL |
268 | if ($v->{ring0_addr} eq $addr || ($v->{ring1_addr} && $v->{ring1_addr} eq $addr)) { |
269 | die "corosync: address '$addr' already defined by node '$k'\n"; | |
270 | } | |
1d26c202 TL |
271 | } |
272 | }; | |
273 | ||
274 | &$check_duplicate_addr($param->{ring0_addr}); | |
275 | &$check_duplicate_addr($param->{ring1_addr}); | |
276 | ||
277 | $param->{ring0_addr} = $name if !$param->{ring0_addr}; | |
278 | ||
279 | die "corosync: using 'ring1_addr' parameter needs a configured ring 1 interface!\n" | |
280 | if $param->{ring1_addr} && !defined($totem_cfg->{interface}->{1}); | |
281 | ||
282 | die "corosync: ring 1 interface configured but 'ring1_addr' parameter not defined!\n" | |
283 | if defined($totem_cfg->{interface}->{1}) && !defined($param->{ring1_addr}); | |
284 | ||
285 | if (defined(my $res = $nodelist->{$name})) { | |
286 | $param->{nodeid} = $res->{nodeid} if !$param->{nodeid}; | |
287 | $param->{votes} = $res->{quorum_votes} if !defined($param->{votes}); | |
288 | ||
289 | if ($res->{quorum_votes} == $param->{votes} && | |
290 | $res->{nodeid} == $param->{nodeid} && $param->{force}) { | |
291 | print "forcing overwrite of configured node '$name'\n"; | |
292 | } else { | |
293 | die "can't add existing node '$name'\n"; | |
294 | } | |
295 | } elsif (!$param->{nodeid}) { | |
296 | my $nodeid = 1; | |
297 | ||
298 | while(1) { | |
299 | my $found = 0; | |
300 | foreach my $v (values %$nodelist) { | |
301 | if ($v->{nodeid} eq $nodeid) { | |
302 | $found = 1; | |
303 | $nodeid++; | |
304 | last; | |
305 | } | |
306 | } | |
307 | last if !$found; | |
308 | }; | |
309 | ||
310 | $param->{nodeid} = $nodeid; | |
311 | } | |
312 | ||
313 | $param->{votes} = 1 if !defined($param->{votes}); | |
314 | ||
315 | PVE::Cluster::gen_local_dirs($name); | |
316 | ||
317 | eval { PVE::Cluster::ssh_merge_keys(); }; | |
318 | warn $@ if $@; | |
319 | ||
320 | $nodelist->{$name} = { | |
321 | ring0_addr => $param->{ring0_addr}, | |
322 | nodeid => $param->{nodeid}, | |
323 | name => $name, | |
324 | }; | |
325 | $nodelist->{$name}->{ring1_addr} = $param->{ring1_addr} if $param->{ring1_addr}; | |
326 | $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes}; | |
327 | ||
855671ec TL |
328 | PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster"); |
329 | ||
1d26c202 TL |
330 | PVE::Corosync::update_nodelist($conf, $nodelist); |
331 | }; | |
332 | ||
333 | $config_change_lock->($code); | |
334 | die $@ if $@; | |
335 | ||
331d957b TL |
336 | my $res = { |
337 | corosync_authkey => PVE::Tools::file_get_contents($authfile), | |
338 | corosync_conf => PVE::Tools::file_get_contents($clusterconf), | |
339 | }; | |
340 | ||
341 | return $res; | |
1d26c202 TL |
342 | }}); |
343 | ||
344 | ||
345 | __PACKAGE__->register_method ({ | |
346 | name => 'delnode', | |
347 | path => 'nodes/{node}', | |
348 | method => 'DELETE', | |
349 | protected => 1, | |
350 | description => "Removes a node from the cluster configuration.", | |
351 | parameters => { | |
352 | additionalProperties => 0, | |
353 | properties => { | |
354 | node => get_standard_option('pve-node'), | |
355 | }, | |
356 | }, | |
357 | returns => { type => 'null' }, | |
358 | code => sub { | |
359 | my ($param) = @_; | |
360 | ||
361 | my $local_node = PVE::INotify::nodename(); | |
362 | die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node; | |
363 | ||
364 | PVE::Cluster::check_cfs_quorum(); | |
365 | ||
366 | my $code = sub { | |
367 | my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); | |
368 | my $nodelist = PVE::Corosync::nodelist($conf); | |
369 | ||
370 | my $node; | |
371 | my $nodeid; | |
372 | ||
373 | foreach my $tmp_node (keys %$nodelist) { | |
374 | my $d = $nodelist->{$tmp_node}; | |
375 | my $ring0_addr = $d->{ring0_addr}; | |
376 | my $ring1_addr = $d->{ring1_addr}; | |
377 | if (($tmp_node eq $param->{node}) || | |
378 | (defined($ring0_addr) && ($ring0_addr eq $param->{node})) || | |
379 | (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) { | |
380 | $node = $tmp_node; | |
381 | $nodeid = $d->{nodeid}; | |
382 | last; | |
383 | } | |
384 | } | |
385 | ||
386 | die "Node/IP: $param->{node} is not a known host of the cluster.\n" | |
387 | if !defined($node); | |
388 | ||
855671ec TL |
389 | PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster"); |
390 | ||
1d26c202 TL |
391 | delete $nodelist->{$node}; |
392 | ||
393 | PVE::Corosync::update_nodelist($conf, $nodelist); | |
394 | ||
395 | PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid]) if defined($nodeid); | |
396 | }; | |
397 | ||
398 | $config_change_lock->($code); | |
399 | die $@ if $@; | |
400 | ||
401 | return undef; | |
402 | }}); | |
403 | ||
fca7797d TL |
404 | __PACKAGE__->register_method ({ |
405 | name => 'join_info', | |
406 | path => 'join', | |
be85d00b TL |
407 | permissions => { |
408 | check => ['perm', '/', [ 'Sys.Audit' ]], | |
409 | }, | |
fca7797d TL |
410 | method => 'GET', |
411 | description => "Get information needed to join this cluster over the connected node.", | |
412 | parameters => { | |
413 | additionalProperties => 0, | |
414 | properties => { | |
415 | node => get_standard_option('pve-node', { | |
416 | description => "The node for which the joinee gets the nodeinfo. ", | |
417 | default => "current connected node", | |
418 | optional => 1, | |
419 | }), | |
420 | }, | |
421 | }, | |
422 | returns => { | |
423 | type => 'object', | |
424 | additionalProperties => 0, | |
425 | properties => { | |
426 | nodelist => { | |
427 | type => 'array', | |
428 | items => { | |
429 | type => "object", | |
430 | additionalProperties => 1, | |
431 | properties => { | |
432 | name => get_standard_option('pve-node'), | |
433 | nodeid => get_standard_option('corosync-nodeid'), | |
434 | ring0_addr => get_standard_option('corosync-ring0-addr'), | |
435 | quorum_votes => { type => 'integer', minimum => 0 }, | |
436 | pve_addr => { type => 'string', format => 'ip' }, | |
437 | pve_fp => get_standard_option('fingerprint-sha256'), | |
438 | }, | |
439 | }, | |
440 | }, | |
441 | preferred_node => get_standard_option('pve-node'), | |
442 | totem => { type => 'object' }, | |
443 | config_digest => { type => 'string' }, | |
444 | }, | |
445 | }, | |
446 | code => sub { | |
447 | my ($param) = @_; | |
448 | ||
449 | my $nodename = $param->{node} // PVE::INotify::nodename(); | |
450 | ||
451 | PVE::Cluster::cfs_update(1); | |
452 | my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); | |
453 | ||
454 | die "node is not in a cluster, no join info available!\n" | |
455 | if !($conf && $conf->{main}); | |
456 | ||
457 | my $totem_cfg = $conf->{main}->{totem} // {}; | |
458 | my $nodelist = $conf->{main}->{nodelist}->{node} // {}; | |
459 | my $corosync_config_digest = $conf->{digest}; | |
460 | ||
461 | die "unknown node '$nodename'\n" if ! $nodelist->{$nodename}; | |
462 | ||
463 | foreach my $name (keys %$nodelist) { | |
464 | my $node = $nodelist->{$name}; | |
465 | $node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name); | |
466 | $node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name)); | |
467 | } | |
468 | ||
469 | my $res = { | |
470 | nodelist => [ values %$nodelist ], | |
471 | preferred_node => $nodename, | |
472 | totem => $totem_cfg, | |
473 | config_digest => $corosync_config_digest, | |
474 | }; | |
475 | ||
476 | return $res; | |
477 | }}); | |
478 | ||
6ed49eb1 TL |
479 | __PACKAGE__->register_method ({ |
480 | name => 'join', | |
481 | path => 'join', | |
482 | method => 'POST', | |
483 | protected => 1, | |
484 | description => "Joins this node into an existing cluster.", | |
485 | parameters => { | |
486 | additionalProperties => 0, | |
487 | properties => { | |
488 | hostname => { | |
489 | type => 'string', | |
490 | description => "Hostname (or IP) of an existing cluster member." | |
491 | }, | |
10c6810e | 492 | nodeid => get_standard_option('corosync-nodeid'), |
6ed49eb1 TL |
493 | votes => { |
494 | type => 'integer', | |
495 | description => "Number of votes for this node", | |
496 | minimum => 0, | |
497 | optional => 1, | |
498 | }, | |
499 | force => { | |
500 | type => 'boolean', | |
501 | description => "Do not throw error if node already exists.", | |
502 | optional => 1, | |
503 | }, | |
10c6810e | 504 | ring0_addr => get_standard_option('corosync-ring0-addr', { |
072ba2e0 | 505 | default => "IP resolved by node's hostname", |
10c6810e TL |
506 | }), |
507 | ring1_addr => get_standard_option('corosync-ring1-addr'), | |
6ed49eb1 TL |
508 | fingerprint => get_standard_option('fingerprint-sha256'), |
509 | password => { | |
510 | description => "Superuser (root) password of peer node.", | |
511 | type => 'string', | |
512 | maxLength => 128, | |
513 | }, | |
514 | }, | |
515 | }, | |
516 | returns => { type => 'string' }, | |
517 | code => sub { | |
518 | my ($param) = @_; | |
519 | ||
520 | my $rpcenv = PVE::RPCEnvironment::get(); | |
521 | my $authuser = $rpcenv->get_user(); | |
522 | ||
523 | my $worker = sub { | |
17a4445f | 524 | STDOUT->autoflush(); |
b184a69d TL |
525 | PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::join, $param); |
526 | die $@ if $@; | |
6ed49eb1 TL |
527 | }; |
528 | ||
8c4e30d3 | 529 | return $rpcenv->fork_worker('clusterjoin', undef, $authuser, $worker); |
6ed49eb1 TL |
530 | }}); |
531 | ||
532 | ||
963c06bb DM |
533 | __PACKAGE__->register_method({ |
534 | name => 'totem', | |
535 | path => 'totem', | |
536 | method => 'GET', | |
537 | description => "Get corosync totem protocol settings.", | |
fb7c665a EK |
538 | permissions => { |
539 | check => ['perm', '/', [ 'Sys.Audit' ]], | |
540 | }, | |
963c06bb DM |
541 | parameters => { |
542 | additionalProperties => 0, | |
543 | properties => {}, | |
544 | }, | |
545 | returns => { | |
546 | type => "object", | |
547 | properties => {}, | |
548 | }, | |
549 | code => sub { | |
550 | my ($param) = @_; | |
551 | ||
552 | ||
553 | my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); | |
554 | ||
496de919 TL |
555 | my $totem_cfg = $conf->{main}->{totem}; |
556 | ||
557 | return $totem_cfg; | |
963c06bb DM |
558 | }}); |
559 | ||
560 | 1; |