]>
Commit | Line | Data |
---|---|---|
98fe93ae DC |
1 | package PVE::API2::Ceph::MON; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | ||
6 | use Net::IP; | |
7 | use File::Path; | |
8 | ||
9 | use PVE::Ceph::Tools; | |
10 | use PVE::Ceph::Services; | |
11 | use PVE::Cluster qw(cfs_read_file cfs_write_file); | |
12 | use PVE::JSONSchema qw(get_standard_option); | |
13 | use PVE::Network; | |
14 | use PVE::RADOS; | |
15 | use PVE::RESTHandler; | |
16 | use PVE::RPCEnvironment; | |
17 | use PVE::Tools qw(run_command file_set_contents); | |
790ebc90 | 18 | use PVE::CephConfig; |
2fb592dd | 19 | use PVE::API2::Ceph::MGR; |
98fe93ae DC |
20 | |
21 | use base qw(PVE::RESTHandler); | |
22 | ||
0b6a2838 AA |
23 | my $find_mon_ips = sub { |
24 | my ($cfg, $rados, $node, $mon_address) = @_; | |
25 | ||
26 | my $overwrite_ips = [ PVE::Tools::split_list($mon_address) ]; | |
27 | $overwrite_ips = PVE::Network::unique_ips($overwrite_ips); | |
7e32d0bd DC |
28 | |
29 | my $pubnet; | |
30 | if ($rados) { | |
31 | $pubnet = $rados->mon_command({ prefix => "config get" , who => "mon.", | |
32 | key => "public_network", format => 'plain' }); | |
33 | # if not defined in the db, the result is empty, it is also always | |
34 | # followed by a newline | |
35 | ($pubnet) = $pubnet =~ m/^(\S+)$/; | |
36 | } | |
37 | $pubnet //= $cfg->{global}->{public_network}; | |
98fe93ae DC |
38 | |
39 | if (!$pubnet) { | |
0b6a2838 AA |
40 | if (scalar(@{$overwrite_ips})) { |
41 | return $overwrite_ips; | |
42 | } else { | |
43 | # don't refactor into '[ PVE::Cluster::remote... ]' as it uses wantarray | |
44 | my $ip = PVE::Cluster::remote_node_ip($node); | |
45 | return [ $ip ]; | |
46 | } | |
98fe93ae DC |
47 | } |
48 | ||
0b6a2838 AA |
49 | my $public_nets = [ PVE::Tools::split_list($pubnet) ]; |
50 | if (scalar(@{$public_nets}) > 1) { | |
51 | warn "Multiple Ceph public networks detected on $node: $pubnet\n"; | |
52 | warn "Networks must be capable of routing to each other.\n"; | |
53 | } | |
54 | ||
55 | my $res = []; | |
56 | ||
57 | if (!scalar(@{$overwrite_ips})) { # auto-select one address for each public network | |
58 | for my $net (@{$public_nets}) { | |
0b6a2838 AA |
59 | my $allowed_ips = PVE::Network::get_local_ip_from_cidr($net); |
60 | $allowed_ips = PVE::Network::unique_ips($allowed_ips); | |
396acb15 | 61 | |
0b6a2838 AA |
62 | die "No active IP found for the requested ceph public network '$net' on node '$node'\n" |
63 | if scalar(@$allowed_ips) < 1; | |
98fe93ae | 64 | |
0b6a2838 AA |
65 | if (scalar(@$allowed_ips) == 1) { |
66 | push @{$res}, $allowed_ips->[0]; | |
67 | } else { | |
68 | die "Multiple IPs for ceph public network '$net' detected on $node:\n". | |
69 | join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n"; | |
70 | } | |
98fe93ae | 71 | } |
0b6a2838 AA |
72 | } else { # check if overwrite IPs are active and in any of the public networks |
73 | my $allowed_list = []; | |
74 | ||
75 | for my $net (@{$public_nets}) { | |
0b6a2838 | 76 | push @{$allowed_list}, @{PVE::Network::get_local_ip_from_cidr($net)}; |
98fe93ae | 77 | } |
98fe93ae | 78 | |
0b6a2838 AA |
79 | my $allowed_ips = PVE::Network::unique_ips($allowed_list); |
80 | ||
81 | for my $overwrite_ip (@{$overwrite_ips}) { | |
82 | die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n" | |
83 | if !grep { $_ eq $overwrite_ip } @{$allowed_ips}; | |
84 | ||
85 | push @{$res}, $overwrite_ip; | |
86 | } | |
98fe93ae | 87 | } |
0b6a2838 AA |
88 | |
89 | return $res; | |
98fe93ae DC |
90 | }; |
91 | ||
4be756f5 FE |
92 | my $ips_from_mon_host = sub { |
93 | my ($mon_host) = @_; | |
94 | ||
95 | my $ips = []; | |
96 | ||
97 | my @hosts = PVE::Tools::split_list($mon_host); | |
98 | ||
99 | for my $host (@hosts) { | |
100 | $host =~ s|^\[?v\d+\:||; # remove beginning of vector | |
101 | $host =~ s|/\d+\]?||; # remove end of vector | |
102 | ||
103 | ($host) = PVE::Tools::parse_host_and_port($host); | |
104 | next if !defined($host); | |
105 | ||
106 | # filter out hostnames | |
107 | my $ip = PVE::JSONSchema::pve_verify_ip($host, 1); | |
108 | next if !defined($ip); | |
109 | ||
110 | push @{$ips}, $ip; | |
111 | } | |
112 | ||
113 | return $ips; | |
114 | }; | |
115 | ||
db1c4cc8 | 116 | my $assert_mon_prerequisites = sub { |
0b6a2838 | 117 | my ($cfg, $monhash, $monid, $monips) = @_; |
db1c4cc8 | 118 | |
9e989449 FE |
119 | my $used_ips = {}; |
120 | ||
121 | my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host}); | |
122 | ||
123 | for my $mon_host_ip (@{$mon_host_ips}) { | |
124 | my $ip = PVE::Network::canonical_ip($mon_host_ip); | |
125 | $used_ips->{$ip} = 1; | |
126 | } | |
127 | ||
128 | for my $mon (values %{$monhash}) { | |
129 | next if !defined($mon->{addr}); | |
130 | ||
e6f55a13 FE |
131 | for my $ip ($ips_from_mon_host->($mon->{addr})->@*) { |
132 | $ip = PVE::Network::canonical_ip($ip); | |
133 | $used_ips->{$ip} = 1; | |
134 | } | |
db1c4cc8 DC |
135 | } |
136 | ||
0b6a2838 AA |
137 | for my $monip (@{$monips}) { |
138 | $monip = PVE::Network::canonical_ip($monip); | |
139 | die "monitor address '$monip' already in use\n" if $used_ips->{$monip}; | |
140 | } | |
9e989449 | 141 | |
db1c4cc8 DC |
142 | if (defined($monhash->{$monid})) { |
143 | die "monitor '$monid' already exists\n"; | |
144 | } | |
145 | }; | |
146 | ||
3babcc1d DC |
147 | my $assert_mon_can_remove = sub { |
148 | my ($monhash, $monlist, $monid, $mondir) = @_; | |
149 | ||
150 | if (!(defined($monhash->{"mon.$monid"}) || | |
151 | grep { $_->{name} && $_->{name} eq $monid } @$monlist)) | |
152 | { | |
153 | die "no such monitor id '$monid'\n" | |
154 | } | |
155 | ||
156 | die "monitor filesystem '$mondir' does not exist on this node\n" if ! -d $mondir; | |
157 | die "can't remove last monitor\n" if scalar(@$monlist) <= 1; | |
158 | }; | |
159 | ||
3e10f0fc FE |
160 | my $remove_addr_from_mon_host = sub { |
161 | my ($monhost, $addr) = @_; | |
162 | ||
815325da FE |
163 | $addr = "[$addr]" if PVE::JSONSchema::pve_verify_ipv6($addr, 1); |
164 | ||
3e10f0fc FE |
165 | # various replaces to remove the ip |
166 | # we always match the beginning or a separator (also at the end) | |
167 | # so we do not accidentally remove a wrong ip | |
168 | # e.g. removing 10.0.0.1 should not remove 10.0.0.101 or 110.0.0.1 | |
169 | ||
170 | # remove vector containing this ip | |
171 | # format is [vX:ip:port/nonce,vY:ip:port/nonce] | |
172 | my $vectorpart_re = "v\\d+:\Q$addr\E:\\d+\\/\\d+"; | |
173 | $monhost =~ s/(^|[ ,;]*)\[$vectorpart_re(?:,$vectorpart_re)*\](?:[ ,;]+|$)/$1/; | |
174 | ||
175 | # ip (+ port) | |
176 | $monhost =~ s/(^|[ ,;]+)\Q$addr\E(?::\d+)?(?:[ ,;]+|$)/$1/; | |
177 | ||
178 | # ipv6 only without brackets | |
179 | if ($addr =~ m/^\[?(.*?:.*?)\]?$/) { | |
180 | $addr = $1; | |
181 | $monhost =~ s/(^|[ ,;]+)\Q$addr\E(?:[ ,;]+|$)/$1/; | |
182 | } | |
183 | ||
184 | # remove trailing separators | |
185 | $monhost =~ s/[ ,;]+$//; | |
186 | ||
187 | return $monhost; | |
188 | }; | |
189 | ||
98fe93ae DC |
190 | __PACKAGE__->register_method ({ |
191 | name => 'listmon', | |
192 | path => '', | |
193 | method => 'GET', | |
194 | description => "Get Ceph monitor list.", | |
195 | proxyto => 'node', | |
196 | protected => 1, | |
197 | permissions => { | |
198 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
199 | }, | |
200 | parameters => { | |
201 | additionalProperties => 0, | |
202 | properties => { | |
203 | node => get_standard_option('pve-node'), | |
204 | }, | |
205 | }, | |
206 | returns => { | |
207 | type => 'array', | |
208 | items => { | |
209 | type => "object", | |
210 | properties => { | |
d5373b7d | 211 | addr => { type => 'string', optional => 1 }, |
b62ba85a AL |
212 | ceph_version => { type => 'string', optional => 1 }, |
213 | ceph_version_short => { type => 'string', optional => 1 }, | |
214 | direxists => { type => 'string', optional => 1 }, | |
215 | host => { type => 'boolean', optional => 1 }, | |
216 | name => { type => 'string' }, | |
217 | quorum => { type => 'boolean', optional => 1 }, | |
218 | rank => { type => 'integer', optional => 1 }, | |
219 | service => { type => 'integer', optional => 1 }, | |
220 | state => { type => 'string', optional => 1 }, | |
98fe93ae DC |
221 | }, |
222 | }, | |
223 | links => [ { rel => 'child', href => "{name}" } ], | |
224 | }, | |
225 | code => sub { | |
226 | my ($param) = @_; | |
227 | ||
228 | PVE::Ceph::Tools::check_ceph_inited(); | |
229 | ||
230 | my $res = []; | |
231 | ||
232 | my $cfg = cfs_read_file('ceph.conf'); | |
233 | ||
78441ff8 DC |
234 | my $rados = eval { PVE::RADOS->new() }; |
235 | warn $@ if $@; | |
236 | my $monhash = PVE::Ceph::Services::get_services_info("mon", $cfg, $rados); | |
98fe93ae | 237 | |
78441ff8 | 238 | if ($rados) { |
e25dda25 | 239 | my $monstat = $rados->mon_command({ prefix => 'quorum_status' }); |
d5373b7d | 240 | |
98fe93ae DC |
241 | my $mons = $monstat->{monmap}->{mons}; |
242 | foreach my $d (@$mons) { | |
243 | next if !defined($d->{name}); | |
7c9f66d0 TL |
244 | my $name = $d->{name}; |
245 | $monhash->{$name}->{rank} = $d->{rank}; | |
246 | $monhash->{$name}->{addr} = $d->{addr}; | |
98fe93ae | 247 | if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) { |
7c9f66d0 TL |
248 | $monhash->{$name}->{quorum} = 1; |
249 | $monhash->{$name}->{state} = 'running'; | |
98fe93ae DC |
250 | } |
251 | } | |
d5373b7d | 252 | |
78441ff8 DC |
253 | } else { |
254 | # we cannot check the status if we do not have a RADOS | |
255 | # object, so set the state to unknown | |
256 | foreach my $monid (sort keys %$monhash) { | |
257 | $monhash->{$monid}->{state} = 'unknown'; | |
258 | } | |
259 | } | |
98fe93ae DC |
260 | |
261 | return PVE::RESTHandler::hash_to_array($monhash, 'name'); | |
262 | }}); | |
263 | ||
264 | __PACKAGE__->register_method ({ | |
265 | name => 'createmon', | |
a435eaf9 | 266 | path => '{monid}', |
98fe93ae DC |
267 | method => 'POST', |
268 | description => "Create Ceph Monitor and Manager", | |
269 | proxyto => 'node', | |
270 | protected => 1, | |
271 | permissions => { | |
272 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
273 | }, | |
274 | parameters => { | |
275 | additionalProperties => 0, | |
276 | properties => { | |
277 | node => get_standard_option('pve-node'), | |
a435eaf9 | 278 | monid => { |
98fe93ae DC |
279 | type => 'string', |
280 | optional => 1, | |
7e98f79e DC |
281 | pattern => PVE::Ceph::Services::SERVICE_REGEX, |
282 | maxLength => 200, | |
98fe93ae DC |
283 | description => "The ID for the monitor, when omitted the same as the nodename", |
284 | }, | |
98fe93ae | 285 | 'mon-address' => { |
0b6a2838 AA |
286 | description => 'Overwrites autodetected monitor IP address(es). ' . |
287 | 'Must be in the public network(s) of Ceph.', | |
288 | type => 'string', format => 'ip-list', | |
98fe93ae DC |
289 | optional => 1, |
290 | }, | |
291 | }, | |
292 | }, | |
293 | returns => { type => 'string' }, | |
294 | code => sub { | |
295 | my ($param) = @_; | |
296 | ||
297 | PVE::Ceph::Tools::check_ceph_installed('ceph_mon'); | |
98fe93ae | 298 | PVE::Ceph::Tools::check_ceph_inited(); |
98fe93ae DC |
299 | PVE::Ceph::Tools::setup_pve_symlinks(); |
300 | ||
301 | my $rpcenv = PVE::RPCEnvironment::get(); | |
98fe93ae DC |
302 | my $authuser = $rpcenv->get_user(); |
303 | ||
304 | my $cfg = cfs_read_file('ceph.conf'); | |
7e32d0bd | 305 | my $rados = eval { PVE::RADOS->new() }; # try a rados connection, fails for first monitor |
db1c4cc8 | 306 | my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados); |
98fe93ae | 307 | |
0dd48804 TL |
308 | my $is_first_monitor = !(scalar(keys %$monhash) || $cfg->{global}->{mon_host}); |
309 | ||
310 | if (!defined($rados) && !$is_first_monitor) { | |
db1c4cc8 | 311 | die "Could not connect to ceph cluster despite configured monitors\n"; |
98fe93ae DC |
312 | } |
313 | ||
a435eaf9 | 314 | my $monid = $param->{monid} // $param->{node}; |
98fe93ae | 315 | my $monsection = "mon.$monid"; |
0b6a2838 | 316 | my $ips = $find_mon_ips->($cfg, $rados, $param->{node}, $param->{'mon-address'}); |
98fe93ae | 317 | |
0b6a2838 | 318 | $assert_mon_prerequisites->($cfg, $monhash, $monid, $ips); |
98fe93ae DC |
319 | |
320 | my $worker = sub { | |
321 | my $upid = shift; | |
322 | ||
ad475e25 DC |
323 | PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub { |
324 | # update cfg content and reassert prereqs inside the lock | |
325 | $cfg = cfs_read_file('ceph.conf'); | |
326 | # reopen with longer timeout | |
327 | if (defined($rados)) { | |
328 | $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout')); | |
329 | } | |
330 | $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados); | |
0b6a2838 | 331 | $assert_mon_prerequisites->($cfg, $monhash, $monid, $ips); |
98fe93ae | 332 | |
ad475e25 DC |
333 | my $client_keyring = PVE::Ceph::Tools::get_or_create_admin_keyring(); |
334 | my $mon_keyring = PVE::Ceph::Tools::get_config('pve_mon_key_path'); | |
335 | ||
336 | if (! -f $mon_keyring) { | |
0dd48804 | 337 | print "creating new monitor keyring\n"; |
6e96b070 FE |
338 | run_command([ |
339 | 'ceph-authtool', | |
340 | '--create-keyring', | |
341 | $mon_keyring, | |
342 | '--gen-key', | |
343 | '-n', | |
344 | 'mon.', | |
345 | '--cap', | |
346 | 'mon', | |
347 | 'allow *', | |
348 | ]); | |
349 | run_command([ | |
350 | 'ceph-authtool', | |
351 | $mon_keyring, | |
352 | '--import-keyring', | |
353 | $client_keyring, | |
354 | ]); | |
ad475e25 | 355 | } |
98fe93ae | 356 | |
ad475e25 DC |
357 | my $ccname = PVE::Ceph::Tools::get_config('ccname'); |
358 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; | |
359 | -d $mondir && die "monitor filesystem '$mondir' already exist\n"; | |
98fe93ae | 360 | |
ad475e25 | 361 | my $monmap = "/tmp/monmap"; |
98fe93ae | 362 | |
ad475e25 DC |
363 | eval { |
364 | mkdir $mondir; | |
98fe93ae | 365 | |
6e96b070 | 366 | run_command(['chown', 'ceph:ceph', $mondir]); |
98fe93ae | 367 | |
d3b899c1 | 368 | my $is_first_address = !defined($rados); |
d3b899c1 | 369 | |
0b6a2838 AA |
370 | my $monaddrs = []; |
371 | ||
372 | for my $ip (@{$ips}) { | |
373 | if (Net::IP::ip_is_ipv6($ip)) { | |
374 | $cfg->{global}->{ms_bind_ipv6} = 'true'; | |
375 | $cfg->{global}->{ms_bind_ipv4} = 'false' if $is_first_address; | |
376 | } else { | |
377 | $cfg->{global}->{ms_bind_ipv4} = 'true'; | |
378 | $cfg->{global}->{ms_bind_ipv6} = 'false' if $is_first_address; | |
379 | } | |
380 | ||
381 | my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]" : $ip; | |
382 | push @{$monaddrs}, "v2:$monaddr:3300"; | |
383 | push @{$monaddrs}, "v1:$monaddr:6789"; | |
384 | ||
385 | $is_first_address = 0; | |
386 | } | |
57951fc7 FE |
387 | |
388 | my $monmaptool_cmd = [ | |
389 | 'monmaptool', | |
57951fc7 FE |
390 | '--clobber', |
391 | '--addv', | |
392 | $monid, | |
0b6a2838 | 393 | "[" . join(',', @{$monaddrs}) . "]", |
57951fc7 FE |
394 | '--print', |
395 | $monmap, | |
396 | ]; | |
397 | ||
ad475e25 DC |
398 | if (defined($rados)) { # we can only have a RADOS object if we have a monitor |
399 | my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' }); | |
400 | file_set_contents($monmap, $mapdata); | |
8ecaa0bf | 401 | run_command($monmaptool_cmd); |
ad475e25 | 402 | } else { # we need to create a monmap for the first monitor |
8ecaa0bf | 403 | push @{$monmaptool_cmd}, '--create'; |
57951fc7 | 404 | run_command($monmaptool_cmd); |
10907e54 | 405 | } |
98fe93ae | 406 | |
6e96b070 FE |
407 | run_command([ |
408 | 'ceph-mon', | |
409 | '--mkfs', | |
410 | '-i', | |
411 | $monid, | |
412 | '--monmap', | |
413 | $monmap, | |
414 | '--keyring', | |
415 | $mon_keyring, | |
416 | ]); | |
417 | run_command(['chown', 'ceph:ceph', '-R', $mondir]); | |
ad475e25 DC |
418 | }; |
419 | my $err = $@; | |
420 | unlink $monmap; | |
421 | if ($err) { | |
422 | File::Path::remove_tree($mondir); | |
423 | die $err; | |
424 | } | |
98fe93ae | 425 | |
ad475e25 DC |
426 | # update ceph.conf |
427 | my $monhost = $cfg->{global}->{mon_host} // ""; | |
351d128f DC |
428 | # add all known monitor ips to mon_host if it does not exist |
429 | if (!defined($cfg->{global}->{mon_host})) { | |
430 | for my $mon (sort keys %$monhash) { | |
431 | $monhost .= " " . $monhash->{$mon}->{addr}; | |
432 | } | |
433 | } | |
0b6a2838 | 434 | $monhost .= " " . join(' ', @{$ips}); |
ad475e25 | 435 | $cfg->{global}->{mon_host} = $monhost; |
485b2cd1 | 436 | # The IP is needed in the ceph.conf for the first boot |
0b6a2838 | 437 | $cfg->{$monsection}->{public_addr} = $ips->[0]; |
98fe93ae | 438 | |
ad475e25 | 439 | cfs_write_file('ceph.conf', $cfg); |
98fe93ae | 440 | |
ad475e25 | 441 | PVE::Ceph::Services::ceph_service_cmd('start', $monsection); |
10907e54 | 442 | |
0dd48804 TL |
443 | if ($is_first_monitor) { |
444 | print "created the first monitor, assume it's safe to disable insecure global" | |
445 | ." ID reclaim for new setup\n"; | |
446 | eval { | |
447 | run_command( | |
448 | ['ceph', 'config', 'set', 'mon', 'auth_allow_insecure_global_id_reclaim', 'false'], | |
449 | errfunc => sub { print STDERR "$_[0]\n" }, | |
450 | ) | |
451 | }; | |
452 | warn "$@" if $@; | |
453 | } | |
454 | ||
ad475e25 DC |
455 | eval { PVE::Ceph::Services::ceph_service_cmd('enable', $monsection) }; |
456 | warn "Enable ceph-mon\@${monid}.service failed, do manually: $@\n" if $@; | |
10907e54 | 457 | |
ad475e25 DC |
458 | PVE::Ceph::Services::broadcast_ceph_services(); |
459 | }); | |
460 | die $@ if $@; | |
2fb592dd | 461 | # automatically create manager after the first monitor is created |
51498a26 | 462 | if ($is_first_monitor) { |
2fb592dd TM |
463 | PVE::API2::Ceph::MGR->createmgr({ |
464 | node => $param->{node}, | |
465 | id => $param->{node} | |
466 | }) | |
467 | } | |
98fe93ae DC |
468 | }; |
469 | ||
470 | return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker); | |
471 | }}); | |
472 | ||
473 | __PACKAGE__->register_method ({ | |
474 | name => 'destroymon', | |
475 | path => '{monid}', | |
476 | method => 'DELETE', | |
477 | description => "Destroy Ceph Monitor and Manager.", | |
478 | proxyto => 'node', | |
479 | protected => 1, | |
480 | permissions => { | |
481 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
482 | }, | |
483 | parameters => { | |
484 | additionalProperties => 0, | |
485 | properties => { | |
486 | node => get_standard_option('pve-node'), | |
487 | monid => { | |
488 | description => 'Monitor ID', | |
489 | type => 'string', | |
7e98f79e | 490 | pattern => PVE::Ceph::Services::SERVICE_REGEX, |
98fe93ae | 491 | }, |
98fe93ae DC |
492 | }, |
493 | }, | |
494 | returns => { type => 'string' }, | |
495 | code => sub { | |
496 | my ($param) = @_; | |
497 | ||
498 | my $rpcenv = PVE::RPCEnvironment::get(); | |
499 | ||
500 | my $authuser = $rpcenv->get_user(); | |
501 | ||
502 | PVE::Ceph::Tools::check_ceph_inited(); | |
503 | ||
504 | my $cfg = cfs_read_file('ceph.conf'); | |
505 | ||
506 | my $monid = $param->{monid}; | |
507 | my $monsection = "mon.$monid"; | |
508 | ||
509 | my $rados = PVE::RADOS->new(); | |
e25dda25 | 510 | my $monstat = $rados->mon_command({ prefix => 'quorum_status' }); |
98fe93ae | 511 | my $monlist = $monstat->{monmap}->{mons}; |
3babcc1d | 512 | my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados); |
98fe93ae DC |
513 | |
514 | my $ccname = PVE::Ceph::Tools::get_config('ccname'); | |
98fe93ae | 515 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; |
98fe93ae | 516 | |
3babcc1d | 517 | $assert_mon_can_remove->($monhash, $monlist, $monid, $mondir); |
98fe93ae DC |
518 | |
519 | my $worker = sub { | |
520 | my $upid = shift; | |
9bc15eea DC |
521 | PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub { |
522 | # reload info and recheck | |
523 | $cfg = cfs_read_file('ceph.conf'); | |
524 | ||
525 | # reopen with longer timeout | |
526 | $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout')); | |
527 | $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados); | |
e25dda25 | 528 | $monstat = $rados->mon_command({ prefix => 'quorum_status' }); |
9bc15eea | 529 | $monlist = $monstat->{monmap}->{mons}; |
98fe93ae | 530 | |
0b6a2838 AA |
531 | my $addrs = []; |
532 | ||
533 | my $add_addr = sub { | |
534 | my ($addr) = @_; | |
535 | ||
536 | # extract the ip without port and nonce (if present) | |
537 | ($addr) = $addr =~ m|^(.*):\d+(/\d+)?$|; | |
538 | ($addr) = $addr =~ m|^\[?(.*?)\]?$|; # remove brackets | |
539 | push @{$addrs}, $addr; | |
540 | }; | |
541 | ||
ea2ecb0c DC |
542 | for my $mon (@$monlist) { |
543 | if ($mon->{name} eq $monid) { | |
0b6a2838 AA |
544 | if ($mon->{public_addrs} && $mon->{public_addrs}->{addrvec}) { |
545 | my $addrvec = $mon->{public_addrs}->{addrvec}; | |
546 | for my $addr (@{$addrvec}) { | |
547 | $add_addr->($addr->{addr}); | |
548 | } | |
549 | } else { | |
550 | $add_addr->($mon->{public_addr} // $mon->{addr}); | |
551 | } | |
ea2ecb0c DC |
552 | last; |
553 | } | |
554 | } | |
555 | ||
9bc15eea | 556 | $assert_mon_can_remove->($monhash, $monlist, $monid, $mondir); |
98fe93ae | 557 | |
f4d09458 | 558 | # this also stops the service |
9bc15eea | 559 | $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' }); |
98fe93ae | 560 | |
9bc15eea DC |
561 | # delete section |
562 | delete $cfg->{$monsection}; | |
563 | ||
ea2ecb0c DC |
564 | # delete from mon_host |
565 | if (my $monhost = $cfg->{global}->{mon_host}) { | |
0b6a2838 | 566 | my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host}); |
815325da | 567 | |
0b6a2838 AA |
568 | for my $addr (@{$addrs}) { |
569 | $monhost = $remove_addr_from_mon_host->($monhost, $addr); | |
815325da | 570 | |
0b6a2838 AA |
571 | # also remove matching IPs that differ syntactically |
572 | if (PVE::JSONSchema::pve_verify_ip($addr, 1)) { | |
573 | $addr = PVE::Network::canonical_ip($addr); | |
815325da | 574 | |
0b6a2838 AA |
575 | for my $mon_host_ip (@{$mon_host_ips}) { |
576 | # match canonical addresses, but remove as present in mon_host | |
577 | if (PVE::Network::canonical_ip($mon_host_ip) eq $addr) { | |
578 | $monhost = $remove_addr_from_mon_host->($monhost, $mon_host_ip); | |
579 | } | |
815325da FE |
580 | } |
581 | } | |
582 | } | |
583 | $cfg->{global}->{mon_host} = $monhost; | |
ea2ecb0c DC |
584 | } |
585 | ||
9bc15eea DC |
586 | cfs_write_file('ceph.conf', $cfg); |
587 | File::Path::remove_tree($mondir); | |
588 | eval { PVE::Ceph::Services::ceph_service_cmd('disable', $monsection) }; | |
589 | warn $@ if $@; | |
590 | PVE::Ceph::Services::broadcast_ceph_services(); | |
591 | }); | |
b4cb37e0 FG |
592 | |
593 | die $@ if $@; | |
98fe93ae DC |
594 | }; |
595 | ||
596 | return $rpcenv->fork_worker('cephdestroymon', $monsection, $authuser, $worker); | |
597 | }}); | |
598 | ||
599 | 1; |