]>
Commit | Line | Data |
---|---|---|
fb5f2d1e | 1 | package PMG::API2::Cluster; |
5ac6bd0e DM |
2 | |
3 | use strict; | |
4 | use warnings; | |
5 | use Data::Dumper; | |
6 | ||
7 | use PVE::SafeSyslog; | |
8 | use PVE::Tools qw(extract_param); | |
9 | use HTTP::Status qw(:constants); | |
10 | use Storable qw(dclone); | |
11 | use PVE::JSONSchema qw(get_standard_option); | |
12 | use PVE::RESTHandler; | |
13 | use PVE::INotify; | |
e16a9efc | 14 | use PVE::APIClient::LWP; |
5ac6bd0e | 15 | |
cfdf6608 | 16 | use PMG::RESTEnvironment; |
5ac6bd0e | 17 | use PMG::ClusterConfig; |
e16a9efc | 18 | use PMG::Cluster; |
cfdf6608 | 19 | use PMG::DBTools; |
f607fc2f | 20 | use PMG::MailQueue; |
5ac6bd0e | 21 | |
ba11e2d3 DM |
22 | use PMG::API2::Nodes; |
23 | ||
5ac6bd0e DM |
24 | use base qw(PVE::RESTHandler); |
25 | ||
e16a9efc | 26 | sub cluster_join { |
245b527c | 27 | my ($cinfo, $conn_setup) = @_; |
e16a9efc DM |
28 | |
29 | my $conn = PVE::APIClient::LWP->new(%$conn_setup); | |
30 | ||
31 | my $info = PMG::Cluster::read_local_cluster_info(); | |
32 | ||
33 | my $res = $conn->post("/config/cluster/nodes", $info); | |
275d6d6d DM |
34 | |
35 | foreach my $node (@$res) { | |
245b527c | 36 | $cinfo->{ids}->{$node->{cid}} = $node; |
275d6d6d DM |
37 | } |
38 | ||
5a312cf1 DM |
39 | eval { |
40 | print STDERR "stop all services accessing the database\n"; | |
41 | # stop all services accessing the database | |
00d8b7f7 | 42 | PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list); |
5a312cf1 DM |
43 | |
44 | print STDERR "save new cluster configuration\n"; | |
245b527c | 45 | $cinfo->write(); |
5a312cf1 | 46 | |
245b527c | 47 | PMG::Cluster::update_ssh_keys($cinfo); |
5a312cf1 DM |
48 | |
49 | print STDERR "cluster node successfully joined\n"; | |
50 | ||
245b527c | 51 | $cinfo = PMG::ClusterConfig->new(); # reload |
5a312cf1 | 52 | |
245b527c DM |
53 | my $role = $cinfo->{'local'}->{type} // '-'; |
54 | die "local node '$cinfo->{local}->{name}' not part of cluster\n" | |
5a312cf1 DM |
55 | if $role eq '-'; |
56 | ||
245b527c | 57 | die "got unexpected role '$role' for local node '$cinfo->{local}->{name}'\n" |
809ae8f4 DM |
58 | if $role ne 'node'; |
59 | ||
245b527c | 60 | my $cid = $cinfo->{'local'}->{cid}; |
5a312cf1 | 61 | |
8293a783 | 62 | PMG::MailQueue::create_spooldirs($cid); |
5a312cf1 | 63 | |
245b527c | 64 | PMG::Cluster::sync_config_from_master($cinfo->{master}->{name}, $cinfo->{master}->{ip}); |
5a312cf1 | 65 | |
245b527c | 66 | PMG::DBTools::init_nodedb($cinfo); |
5a312cf1 | 67 | |
00d8b7f7 | 68 | my $cfg = PMG::Config->new(); |
245b527c DM |
69 | my $ruledb = PMG::RuleDB->new(); |
70 | my $rulecache = PMG::RuleCache->new($ruledb); | |
71 | ||
72 | $cfg->rewrite_config($rulecache, 1); | |
5a312cf1 DM |
73 | |
74 | print STDERR "syncing quarantine data\n"; | |
245b527c | 75 | PMG::Cluster::sync_master_quar($cinfo->{master}->{ip}, $cinfo->{master}->{name}); |
5a312cf1 DM |
76 | print STDERR "syncing quarantine data finished\n"; |
77 | }; | |
78 | my $err = $@; | |
79 | ||
00d8b7f7 | 80 | foreach my $service (reverse @$PMG::Utils::db_service_list) { |
5a312cf1 DM |
81 | eval { PVE::Tools::run_command(['systemctl', 'start', $service]); }; |
82 | warn $@ if $@; | |
83 | } | |
84 | ||
85 | die $err if $err; | |
e16a9efc DM |
86 | } |
87 | ||
5ac6bd0e DM |
88 | __PACKAGE__->register_method({ |
89 | name => 'index', | |
90 | path => '', | |
91 | method => 'GET', | |
fb5f2d1e DM |
92 | description => "Directory index.", |
93 | permissions => { user => 'all' }, | |
94 | parameters => { | |
95 | additionalProperties => 0, | |
96 | properties => {}, | |
97 | }, | |
98 | returns => { | |
99 | type => 'array', | |
100 | items => { | |
101 | type => "object", | |
102 | properties => {}, | |
103 | }, | |
104 | links => [ { rel => 'child', href => "{name}" } ], | |
105 | }, | |
106 | code => sub { | |
107 | my ($param) = @_; | |
108 | ||
109 | my $result = [ | |
110 | { name => 'nodes' }, | |
150e8421 | 111 | { name => 'status' }, |
fb5f2d1e DM |
112 | { name => 'create' }, |
113 | { name => 'join' }, | |
54bc8ad6 | 114 | { name => 'update-fingerprints' }, |
fb5f2d1e DM |
115 | ]; |
116 | ||
117 | return $result; | |
118 | }}); | |
119 | ||
120 | __PACKAGE__->register_method({ | |
121 | name => 'nodes', | |
122 | path => 'nodes', | |
123 | method => 'GET', | |
5ac6bd0e DM |
124 | description => "Cluster node index.", |
125 | # alway read local file | |
126 | parameters => { | |
127 | additionalProperties => 0, | |
128 | properties => {}, | |
129 | }, | |
f4302160 | 130 | permissions => { check => [ 'admin', 'qmanager', 'audit' ] }, |
5ac6bd0e DM |
131 | returns => { |
132 | type => 'array', | |
133 | items => { | |
134 | type => "object", | |
135 | properties => { | |
150e8421 | 136 | type => { type => 'string' }, |
5ac6bd0e | 137 | cid => { type => 'integer' }, |
150e8421 DM |
138 | ip => { type => 'string' }, |
139 | name => { type => 'string' }, | |
140 | hostrsapubkey => { type => 'string' }, | |
141 | rootrsapubkey => { type => 'string' }, | |
142 | fingerprint => { type => 'string' }, | |
143 | }, | |
144 | }, | |
150e8421 DM |
145 | }, |
146 | code => sub { | |
147 | my ($param) = @_; | |
148 | ||
245b527c | 149 | my $cinfo = PMG::ClusterConfig->new(); |
150e8421 | 150 | |
245b527c DM |
151 | if (scalar(keys %{$cinfo->{ids}})) { |
152 | my $role = $cinfo->{local}->{type} // '-'; | |
150e8421 | 153 | if ($role eq '-') { |
245b527c | 154 | die "local node '$cinfo->{local}->{name}' not part of cluster\n"; |
150e8421 DM |
155 | } |
156 | } | |
157 | ||
245b527c | 158 | return PVE::RESTHandler::hash_to_array($cinfo->{ids}, 'cid'); |
150e8421 DM |
159 | }}); |
160 | ||
161 | __PACKAGE__->register_method({ | |
162 | name => 'status', | |
163 | path => 'status', | |
164 | method => 'GET', | |
165 | description => "Cluster node status.", | |
166 | # alway read local file | |
167 | parameters => { | |
168 | additionalProperties => 0, | |
64118ca4 DM |
169 | properties => { |
170 | list_single_node => { | |
171 | description => "List local node if there is no cluster defined. Please note that RSA keys and fingerprint are not valid in that case.", | |
172 | type => 'boolean', | |
173 | optional => 1, | |
174 | default => 0, | |
175 | }, | |
176 | }, | |
150e8421 | 177 | }, |
f4302160 | 178 | permissions => { check => [ 'admin', 'qmanager', 'audit' ] }, |
150e8421 DM |
179 | returns => { |
180 | type => 'array', | |
181 | items => { | |
182 | type => "object", | |
183 | properties => { | |
184 | type => { type => 'string' }, | |
185 | cid => { type => 'integer' }, | |
186 | ip => { type => 'string' }, | |
187 | name => { type => 'string' }, | |
188 | hostrsapubkey => { type => 'string' }, | |
189 | rootrsapubkey => { type => 'string' }, | |
190 | fingerprint => { type => 'string' }, | |
5ac6bd0e DM |
191 | }, |
192 | }, | |
193 | links => [ { rel => 'child', href => "{cid}" } ], | |
194 | }, | |
195 | code => sub { | |
196 | my ($param) = @_; | |
197 | ||
245b527c | 198 | my $cinfo = PMG::ClusterConfig->new(); |
64118ca4 | 199 | my $nodename = PVE::INotify::nodename(); |
5ac6bd0e | 200 | |
64118ca4 | 201 | my $res = []; |
245b527c DM |
202 | if (scalar(keys %{$cinfo->{ids}})) { |
203 | my $role = $cinfo->{local}->{type} // '-'; | |
3862c23d | 204 | if ($role eq '-') { |
245b527c | 205 | die "local node '$cinfo->{local}->{name}' not part of cluster\n"; |
3862c23d | 206 | } |
64118ca4 | 207 | $res = PVE::RESTHandler::hash_to_array($cinfo->{ids}, 'cid'); |
3862c23d | 208 | |
64118ca4 DM |
209 | } elsif ($param->{list_single_node}) { |
210 | my $ni = { type => '-' }; | |
211 | foreach my $k (qw(ip name cid)) { | |
212 | $ni->{$k} = $cinfo->{local}->{$k}; | |
213 | } | |
214 | foreach my $k (qw(hostrsapubkey rootrsapubkey fingerprint)) { | |
215 | $ni->{$k} = '-'; # invalid | |
216 | } | |
217 | $res = [ $ni ]; | |
218 | } | |
ba11e2d3 DM |
219 | |
220 | my $rpcenv = PMG::RESTEnvironment->get(); | |
221 | my $authuser = $rpcenv->get_user(); | |
222 | my $ticket = $rpcenv->get_ticket(); | |
223 | ||
224 | foreach my $ni (@$res) { | |
225 | my $info; | |
226 | eval { | |
245b527c | 227 | if ($ni->{cid} eq $cinfo->{local}->{cid}) { |
64118ca4 | 228 | $info = PMG::API2::NodeInfo->status({ node => $nodename }); |
ba11e2d3 DM |
229 | } else { |
230 | my $conn = PVE::APIClient::LWP->new( | |
231 | ticket => $ticket, | |
232 | cookie_name => 'PMGAuthCookie', | |
233 | host => $ni->{ip}, | |
234 | cached_fingerprints => { | |
235 | $ni->{fingerprint} => 1, | |
236 | }); | |
237 | ||
238 | $info = $conn->get("/nodes/localhost/status", {}); | |
239 | } | |
240 | }; | |
241 | if (my $err = $@) { | |
15eac67d | 242 | $ni->{conn_error} = "$err"; # convert $err to string |
ba11e2d3 DM |
243 | next; |
244 | } | |
245 | foreach my $k (keys %$info) { | |
246 | $ni->{$k} = $info->{$k} if !defined($ni->{$k}); | |
247 | } | |
248 | } | |
249 | ||
250 | return $res; | |
5ac6bd0e DM |
251 | }}); |
252 | ||
e16a9efc DM |
253 | my $add_node_schema = PMG::ClusterConfig::Node->createSchema(1); |
254 | delete $add_node_schema->{properties}->{cid}; | |
255 | ||
256 | __PACKAGE__->register_method({ | |
257 | name => 'add_node', | |
258 | path => 'nodes', | |
259 | method => 'POST', | |
260 | description => "Add an node to the cluster config.", | |
261 | proxyto => 'master', | |
262 | protected => 1, | |
263 | parameters => $add_node_schema, | |
275d6d6d DM |
264 | returns => { |
265 | description => "Returns the resulting node list.", | |
266 | type => 'array', | |
267 | items => { | |
268 | type => "object", | |
269 | properties => { | |
270 | cid => { type => 'integer' }, | |
271 | }, | |
272 | }, | |
273 | }, | |
e16a9efc DM |
274 | code => sub { |
275 | my ($param) = @_; | |
276 | ||
277 | my $code = sub { | |
245b527c | 278 | my $cinfo = PMG::ClusterConfig->new(); |
e16a9efc | 279 | |
245b527c | 280 | die "no cluster defined\n" if !scalar(keys %{$cinfo->{ids}}); |
e16a9efc | 281 | |
245b527c | 282 | my $master = $cinfo->{master} || die "unable to lookup master node\n"; |
e16a9efc | 283 | |
92c560f7 | 284 | my $next_cid; |
245b527c DM |
285 | foreach my $cid (keys %{$cinfo->{ids}}) { |
286 | my $d = $cinfo->{ids}->{$cid}; | |
92c560f7 DM |
287 | |
288 | if ($d->{type} eq 'node' && $d->{ip} eq $param->{ip} && $d->{name} eq $param->{name}) { | |
275d6d6d | 289 | $next_cid = $cid; # allow overwrite existing node data |
92c560f7 DM |
290 | last; |
291 | } | |
292 | ||
293 | if ($d->{ip} eq $param->{ip}) { | |
294 | die "ip address '$param->{ip}' is already used by existing node $d->{name}\n"; | |
295 | } | |
296 | ||
297 | if ($d->{name} eq $param->{name}) { | |
298 | die "node with name '$param->{name}' already exists\n"; | |
299 | } | |
300 | } | |
301 | ||
302 | if (!defined($next_cid)) { | |
303 | $next_cid = ++$master->{maxcid}; | |
304 | } | |
e16a9efc | 305 | |
4d772625 SI |
306 | # create spooldir for new node to prevent problems if it gets |
307 | # delete from the cluster before being synced initially | |
308 | PMG::MailQueue::create_spooldirs($master->{maxcid}); | |
309 | ||
e16a9efc DM |
310 | my $node = { |
311 | type => 'node', | |
312 | cid => $master->{maxcid}, | |
313 | }; | |
314 | ||
315 | foreach my $k (qw(ip name hostrsapubkey rootrsapubkey fingerprint)) { | |
316 | $node->{$k} = $param->{$k}; | |
317 | } | |
318 | ||
245b527c | 319 | $cinfo->{ids}->{$node->{cid}} = $node; |
e16a9efc | 320 | |
245b527c | 321 | $cinfo->write(); |
e16a9efc | 322 | |
5a312cf1 DM |
323 | PMG::DBTools::update_master_clusterinfo($node->{cid}); |
324 | ||
245b527c | 325 | PMG::Cluster::update_ssh_keys($cinfo); |
5a312cf1 | 326 | |
245b527c | 327 | return PVE::RESTHandler::hash_to_array($cinfo->{ids}, 'cid'); |
275d6d6d | 328 | }; |
e16a9efc | 329 | |
275d6d6d | 330 | return PMG::ClusterConfig::lock_config($code, "add node failed"); |
e16a9efc DM |
331 | }}); |
332 | ||
cba17aeb | 333 | __PACKAGE__->register_method({ |
fb5f2d1e DM |
334 | name => 'create', |
335 | path => 'create', | |
cba17aeb DM |
336 | method => 'POST', |
337 | description => "Create initial cluster config with current node as master.", | |
338 | # alway read local file | |
339 | parameters => { | |
340 | additionalProperties => 0, | |
341 | properties => {}, | |
342 | }, | |
cfdf6608 DM |
343 | protected => 1, |
344 | returns => { type => 'string' }, | |
cba17aeb DM |
345 | code => sub { |
346 | my ($param) = @_; | |
347 | ||
cfdf6608 DM |
348 | my $rpcenv = PMG::RESTEnvironment->get(); |
349 | my $authuser = $rpcenv->get_user(); | |
350 | ||
351 | my $realcmd = sub { | |
245b527c | 352 | my $cinfo = PMG::ClusterConfig->new(); |
cba17aeb | 353 | |
56001f4f | 354 | die "cluster already defined\n" if scalar(keys %{$cinfo->{ids}}); |
cba17aeb DM |
355 | |
356 | my $info = PMG::Cluster::read_local_cluster_info(); | |
357 | ||
cfdf6608 DM |
358 | my $cid = 1; |
359 | ||
cba17aeb | 360 | $info->{type} = 'master'; |
cba17aeb | 361 | |
cfdf6608 | 362 | $info->{maxcid} = $cid, |
cba17aeb | 363 | |
245b527c | 364 | $cinfo->{ids}->{$cid} = $info; |
cfdf6608 | 365 | |
cfdf6608 DM |
366 | eval { |
367 | print STDERR "stop all services accessing the database\n"; | |
368 | # stop all services accessing the database | |
00d8b7f7 | 369 | PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list); |
cfdf6608 DM |
370 | |
371 | print STDERR "save new cluster configuration\n"; | |
245b527c | 372 | $cinfo->write(); |
cfdf6608 DM |
373 | |
374 | PMG::DBTools::init_masterdb($cid); | |
375 | ||
8293a783 | 376 | PMG::MailQueue::create_spooldirs($cid); |
cfdf6608 DM |
377 | |
378 | print STDERR "cluster master successfully created\n"; | |
379 | }; | |
380 | my $err = $@; | |
381 | ||
00d8b7f7 | 382 | foreach my $service (reverse @$PMG::Utils::db_service_list) { |
cfdf6608 DM |
383 | eval { PVE::Tools::run_command(['systemctl', 'start', $service]); }; |
384 | warn $@ if $@; | |
385 | } | |
386 | ||
387 | die $err if $err; | |
cba17aeb DM |
388 | }; |
389 | ||
cfdf6608 DM |
390 | my $code = sub { |
391 | return $rpcenv->fork_worker('clustercreate', undef, $authuser, $realcmd); | |
392 | }; | |
cba17aeb | 393 | |
cfdf6608 | 394 | return PMG::ClusterConfig::lock_config($code, "create cluster failed"); |
cba17aeb DM |
395 | }}); |
396 | ||
fb5f2d1e DM |
397 | __PACKAGE__->register_method({ |
398 | name => 'join', | |
399 | path => 'join', | |
400 | method => 'POST', | |
401 | description => "Join local node to an existing cluster.", | |
402 | # alway read local file | |
4884557f | 403 | protected => 1, |
fb5f2d1e DM |
404 | parameters => { |
405 | additionalProperties => 0, | |
406 | properties => { | |
407 | master_ip => { | |
408 | description => "IP address.", | |
409 | type => 'string', format => 'ip', | |
410 | }, | |
411 | fingerprint => { | |
412 | description => "SSL certificate fingerprint.", | |
413 | type => 'string', | |
414 | pattern => '^(:?[A-Z0-9][A-Z0-9]:){31}[A-Z0-9][A-Z0-9]$', | |
415 | }, | |
e16a9efc DM |
416 | password => { |
417 | description => "Superuser password.", | |
418 | type => 'string', | |
419 | maxLength => 128, | |
420 | }, | |
fb5f2d1e DM |
421 | }, |
422 | }, | |
5a312cf1 | 423 | returns => { type => 'string' }, |
fb5f2d1e DM |
424 | code => sub { |
425 | my ($param) = @_; | |
426 | ||
5a312cf1 DM |
427 | my $rpcenv = PMG::RESTEnvironment->get(); |
428 | my $authuser = $rpcenv->get_user(); | |
429 | ||
430 | my $realcmd = sub { | |
245b527c | 431 | my $cinfo = PMG::ClusterConfig->new(); |
fb5f2d1e | 432 | |
56001f4f | 433 | die "cluster already defined\n" if scalar(keys %{$cinfo->{ids}}); |
fb5f2d1e | 434 | |
e16a9efc DM |
435 | my $setup = { |
436 | username => 'root@pam', | |
437 | password => $param->{password}, | |
438 | cookie_name => 'PMGAuthCookie', | |
439 | host => $param->{master_ip}, | |
440 | cached_fingerprints => { | |
441 | $param->{fingerprint} => 1, | |
442 | } | |
443 | }; | |
444 | ||
245b527c | 445 | cluster_join($cinfo, $setup); |
fb5f2d1e DM |
446 | }; |
447 | ||
5a312cf1 DM |
448 | my $code = sub { |
449 | return $rpcenv->fork_worker('clusterjoin', undef, $authuser, $realcmd); | |
450 | }; | |
fb5f2d1e | 451 | |
5a312cf1 | 452 | return PMG::ClusterConfig::lock_config($code, "cluster join failed"); |
fb5f2d1e DM |
453 | }}); |
454 | ||
54bc8ad6 SI |
455 | __PACKAGE__->register_method({ |
456 | name => 'update_fingerprints', | |
457 | path => 'update-fingerprints', | |
458 | method => 'POST', | |
459 | description => "Update API certificate fingerprints (by fetching it via ssh).", | |
460 | proxyto => 'master', | |
461 | protected => 1, | |
462 | parameters => { | |
463 | additionalProperties => 0, | |
464 | }, | |
465 | returns => { type => 'null' }, | |
466 | code => sub { | |
467 | my ($param) = @_; | |
468 | ||
469 | my $code = sub { | |
470 | my $cinfo = PMG::ClusterConfig->new(); | |
471 | ||
472 | die "no cluster defined\n" if !scalar(keys %{$cinfo->{ids}}); | |
473 | ||
474 | my $localcid = $cinfo->{local}->{cid}; | |
475 | ||
476 | foreach my $cid (keys %{$cinfo->{ids}}) { | |
477 | my $d = $cinfo->{ids}->{$cid}; | |
478 | my $fp; | |
479 | if ($d->{cid} == $localcid) { | |
480 | $fp = PMG::Cluster::read_local_ssl_cert_fingerprint(); | |
481 | } else { | |
482 | $fp = PMG::Cluster::get_remote_cert_fingerprint($d); | |
483 | } | |
484 | $cinfo->{ids}->{$d->{cid}}->{fingerprint} = $fp; | |
485 | } | |
486 | ||
487 | $cinfo->write(); | |
488 | ||
489 | return; | |
490 | }; | |
491 | ||
492 | PMG::ClusterConfig::lock_config($code, "update fingerprints failed"); | |
493 | }}); | |
cba17aeb | 494 | |
5ac6bd0e | 495 | 1; |