]>
Commit | Line | Data |
---|---|---|
38db610a DM |
1 | package PVE::API2::Ceph; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | use File::Basename; | |
6 | use File::Path; | |
7 | use POSIX qw (LONG_MAX); | |
8 | ||
9 | use PVE::SafeSyslog; | |
10 | use PVE::Tools qw(extract_param run_command); | |
11 | use PVE::Exception qw(raise raise_param_exc); | |
12 | use PVE::INotify; | |
13 | use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file); | |
14 | use PVE::AccessControl; | |
15 | use PVE::Storage; | |
16 | use PVE::RESTHandler; | |
17 | use PVE::RPCEnvironment; | |
18 | use PVE::JSONSchema qw(get_standard_option); | |
19 | use JSON; | |
20 | ||
21 | use base qw(PVE::RESTHandler); | |
22 | ||
23 | use Data::Dumper; # fixme: remove | |
24 | ||
25 | my $ccname = 'ceph'; # ceph cluster name | |
26 | my $ceph_cfgdir = "/etc/ceph"; | |
27 | my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf"; | |
28 | my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf"; | |
29 | my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring"; | |
30 | my $ceph_mon_key_path = "$ceph_cfgdir/$ccname.mon.keyring"; | |
31 | my $pve_ckeyring_path = "/etc/pve/priv/$ccname.keyring"; | |
32 | my $ceph_ckeyring_path = "$ceph_cfgdir/$ccname.client.admin.keyring"; | |
33 | ||
34 | my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring"; | |
35 | my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring"; | |
36 | ||
37 | my $ceph_bin = "/usr/bin/ceph"; | |
38 | ||
39 | sub purge_all_ceph_files { | |
40 | # fixme: this is very dangerous - should we really support this function? | |
41 | ||
42 | unlink $ceph_cfgpath; | |
43 | unlink $ceph_mon_key_path; | |
44 | unlink $ceph_ckeyring_path; | |
45 | ||
46 | unlink $pve_ceph_cfgpath; | |
47 | unlink $pve_ckeyring_path; | |
48 | unlink $pve_mon_key_path; | |
49 | ||
50 | unlink $ceph_bootstrap_osd_keyring; | |
51 | unlink $ceph_bootstrap_mds_keyring; | |
52 | ||
53 | system("rm -rf /var/lib/ceph/mon/ceph-*"); | |
54 | ||
55 | # remove osd? | |
56 | ||
57 | } | |
58 | ||
59 | my $check_ceph_installed = sub { | |
60 | my ($noerr) = @_; | |
61 | ||
62 | if (! -x $ceph_bin) { | |
63 | die "ceph binaries not installed\n" if !$noerr; | |
64 | return undef; | |
65 | } | |
66 | ||
67 | return 1; | |
68 | }; | |
69 | ||
70 | my $check_ceph_inited = sub { | |
71 | my ($noerr) = @_; | |
72 | ||
73 | return undef if !&$check_ceph_installed($noerr); | |
74 | ||
75 | if (! -f $pve_ceph_cfgpath) { | |
76 | die "pveceph configuration not initialized\n" if !$noerr; | |
77 | return undef; | |
78 | } | |
79 | ||
80 | return 1; | |
81 | }; | |
82 | ||
9662c195 DM |
83 | my $check_ceph_enabled = sub { |
84 | my ($noerr) = @_; | |
85 | ||
86 | return undef if !&$check_ceph_inited($noerr); | |
87 | ||
88 | if (! -f $ceph_cfgpath) { | |
89 | die "pveceph configuration not enabled\n" if !$noerr; | |
90 | return undef; | |
91 | } | |
92 | ||
93 | return 1; | |
94 | }; | |
95 | ||
38db610a DM |
96 | my $force_symlink = sub { |
97 | my ($old, $new) = @_; | |
98 | ||
99 | return if (-l $new) && (readlink($new) eq $old); | |
100 | ||
101 | unlink $new; | |
102 | symlink($old, $new) || | |
103 | die "unable to create symlink '$new' - $!\n"; | |
104 | }; | |
105 | ||
106 | my $parse_ceph_config = sub { | |
107 | my ($filename) = @_; | |
108 | ||
109 | my $cfg = {}; | |
110 | ||
111 | return $cfg if ! -f $filename; | |
112 | ||
113 | my $fh = IO::File->new($filename, "r") || | |
114 | die "unable to open '$filename' - $!\n"; | |
115 | ||
116 | my $section; | |
117 | ||
118 | while (defined(my $line = <$fh>)) { | |
119 | $line =~ s/[;#].*$//; | |
120 | $line =~ s/^\s+//; | |
121 | $line =~ s/\s+$//; | |
122 | next if !$line; | |
123 | ||
124 | $section = $1 if $line =~ m/^\[(\S+)\]$/; | |
125 | if (!$section) { | |
126 | warn "no section - skip: $line\n"; | |
127 | next; | |
128 | } | |
129 | ||
130 | if ($line =~ m/^(.*\S)\s*=\s*(\S.*)$/) { | |
131 | $cfg->{$section}->{$1} = $2; | |
132 | } | |
133 | ||
134 | } | |
135 | ||
136 | return $cfg; | |
137 | }; | |
138 | ||
139 | my $run_ceph_cmd = sub { | |
140 | my ($cmd, %params) = @_; | |
141 | ||
142 | my $timeout = 3; | |
143 | ||
144 | my $oldalarm; | |
145 | eval { | |
146 | local $SIG{ALRM} = sub { die "timeout\n" }; | |
147 | $oldalarm = alarm($timeout); | |
148 | # Note: --connect-timeout does not work with current version | |
149 | # '--connect-timeout', $timeout, | |
150 | ||
151 | run_command(['ceph', '-c', $ceph_cfgpath, @$cmd], %params); | |
152 | alarm(0); | |
153 | }; | |
154 | my $err = $@; | |
155 | ||
156 | alarm($oldalarm) if $oldalarm; | |
157 | ||
158 | die $err if $err; | |
38db610a DM |
159 | }; |
160 | ||
9662c195 DM |
161 | my $run_ceph_cmd_json = sub { |
162 | my ($cmd, %opts) = @_; | |
38db610a DM |
163 | |
164 | my $json = ''; | |
9662c195 DM |
165 | |
166 | my $quiet = delete $opts{quiet}; | |
167 | ||
38db610a DM |
168 | my $parser = sub { |
169 | my $line = shift; | |
170 | $json .= $line; | |
171 | }; | |
172 | ||
173 | my $errfunc = sub { | |
174 | my $line = shift; | |
175 | print "$line\n" if !$quiet; | |
176 | }; | |
177 | ||
9662c195 DM |
178 | &$run_ceph_cmd([@$cmd, '--format', 'json'], |
179 | outfunc => $parser, errfunc => $errfunc); | |
38db610a DM |
180 | |
181 | my $res = decode_json($json); | |
182 | ||
183 | return $res; | |
9662c195 | 184 | }; |
38db610a | 185 | |
9662c195 | 186 | sub ceph_mon_status { |
38db610a | 187 | my ($quiet) = @_; |
9662c195 DM |
188 | |
189 | return &$run_ceph_cmd_json(['mon_status'], quiet => $quiet); | |
38db610a | 190 | |
9662c195 | 191 | } |
38db610a | 192 | |
9662c195 DM |
193 | my $ceph_osd_status = sub { |
194 | my ($quiet) = @_; | |
38db610a | 195 | |
9662c195 | 196 | return &$run_ceph_cmd_json(['osd', 'dump'], quiet => $quiet); |
38db610a DM |
197 | }; |
198 | ||
199 | my $write_ceph_config = sub { | |
200 | my ($cfg) = @_; | |
201 | ||
202 | my $out = ''; | |
203 | foreach my $section (keys %$cfg) { | |
204 | $out .= "[$section]\n"; | |
205 | foreach my $key (sort keys %{$cfg->{$section}}) { | |
206 | $out .= "\t $key = $cfg->{$section}->{$key}\n"; | |
207 | } | |
208 | $out .= "\n"; | |
209 | } | |
210 | ||
211 | PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out); | |
212 | }; | |
213 | ||
214 | my $setup_pve_symlinks = sub { | |
215 | # fail if we find a real file instead of a link | |
216 | if (-f $ceph_cfgpath) { | |
217 | my $lnk = readlink($ceph_cfgpath); | |
218 | die "file '$ceph_cfgpath' already exists\n" | |
219 | if !$lnk || $lnk ne $pve_ceph_cfgpath; | |
220 | } | |
221 | ||
222 | # now assume we are allowed to setup/overwrite content | |
223 | &$force_symlink($pve_ceph_cfgpath, $ceph_cfgpath); | |
224 | &$force_symlink($pve_mon_key_path, $ceph_mon_key_path); | |
225 | &$force_symlink($pve_ckeyring_path, $ceph_ckeyring_path); | |
226 | }; | |
227 | ||
228 | my $ceph_service_cmd = sub { | |
229 | run_command(['service', 'ceph', '-c', $ceph_cfgpath, @_]); | |
230 | }; | |
231 | ||
232 | __PACKAGE__->register_method ({ | |
233 | name => 'index', | |
234 | path => '', | |
235 | method => 'GET', | |
236 | description => "Directory index.", | |
237 | permissions => { user => 'all' }, | |
238 | parameters => { | |
239 | additionalProperties => 0, | |
240 | properties => { | |
241 | node => get_standard_option('pve-node'), | |
242 | }, | |
243 | }, | |
244 | returns => { | |
245 | type => 'array', | |
246 | items => { | |
247 | type => "object", | |
248 | properties => {}, | |
249 | }, | |
250 | links => [ { rel => 'child', href => "{name}" } ], | |
251 | }, | |
252 | code => sub { | |
253 | my ($param) = @_; | |
254 | ||
255 | my $result = [ | |
256 | { name => 'init' }, | |
257 | { name => 'createmon' }, | |
258 | { name => 'destroymon' }, | |
259 | { name => 'createosd' }, | |
260 | { name => 'destroyosd' }, | |
261 | { name => 'stop' }, | |
262 | { name => 'start' }, | |
263 | { name => 'status' }, | |
264 | ]; | |
265 | ||
266 | return $result; | |
267 | }}); | |
268 | ||
269 | __PACKAGE__->register_method ({ | |
270 | name => 'init', | |
271 | path => 'init', | |
272 | method => 'POST', | |
273 | description => "Create initial ceph configuration.", | |
274 | proxyto => 'node', | |
275 | protected => 1, | |
276 | parameters => { | |
277 | additionalProperties => 0, | |
278 | properties => { | |
279 | node => get_standard_option('pve-node'), | |
280 | size => { | |
281 | description => 'Number of replicas per object', | |
282 | type => 'integer', | |
283 | default => 2, | |
284 | optional => 1, | |
285 | minimum => 1, | |
286 | maximum => 3, | |
287 | }, | |
288 | pg_bits => { | |
289 | description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)", | |
290 | type => 'integer', | |
291 | default => 9, | |
292 | optional => 1, | |
293 | minimum => 6, | |
294 | maximum => 14, | |
295 | }, | |
296 | }, | |
297 | }, | |
298 | returns => { type => 'null' }, | |
299 | code => sub { | |
300 | my ($param) = @_; | |
301 | ||
302 | &$check_ceph_installed(); | |
303 | ||
304 | -f $pve_ceph_cfgpath && | |
305 | die "configuration file '$pve_ceph_cfgpath' already exists.\n"; | |
306 | ||
307 | my $pg_bits = $param->{pg_bits} || 9; | |
308 | my $size = $param->{size} || 2; | |
309 | ||
310 | my $global = { | |
311 | 'auth supported' => 'cephx', | |
312 | 'auth cluster required' => 'cephx', | |
313 | 'auth service required' => 'cephx', | |
314 | 'auth client required' => 'cephx', | |
315 | 'filestore xattr use omap' => 'true', | |
316 | 'osd journal size' => '1024', | |
317 | 'osd pool default size' => $size, | |
318 | 'osd pool default min size' => 1, | |
319 | 'osd pg bits' => $pg_bits, | |
320 | 'osd pgp bits' => $pg_bits, | |
321 | }; | |
322 | ||
323 | # this does not work for default pools | |
324 | #'osd pool default pg num' => $pg_num, | |
325 | #'osd pool default pgp num' => $pg_num, | |
326 | ||
327 | &$write_ceph_config({global => $global}); | |
328 | ||
329 | &$setup_pve_symlinks(); | |
330 | ||
331 | return undef; | |
332 | }}); | |
333 | ||
334 | __PACKAGE__->register_method ({ | |
335 | name => 'createmon', | |
336 | path => 'createmon', | |
337 | method => 'POST', | |
338 | description => "Create Ceph Monitor", | |
339 | proxyto => 'node', | |
340 | protected => 1, | |
341 | parameters => { | |
342 | additionalProperties => 0, | |
343 | properties => { | |
344 | node => get_standard_option('pve-node'), | |
345 | }, | |
346 | }, | |
347 | returns => { type => 'null' }, | |
348 | code => sub { | |
349 | my ($param) = @_; | |
350 | ||
351 | &$check_ceph_inited(); | |
352 | ||
353 | &$setup_pve_symlinks(); | |
354 | ||
355 | if (! -f $pve_ckeyring_path) { | |
356 | run_command("ceph-authtool $pve_ckeyring_path --create-keyring " . | |
357 | "--gen-key -n client.admin"); | |
358 | } | |
359 | ||
360 | if (! -f $pve_mon_key_path) { | |
361 | run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); | |
362 | run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " . | |
363 | "--cap mds 'allow *' " . | |
364 | "--cap osd 'allow *' " . | |
365 | "--cap mon 'allow *'"); | |
366 | run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); | |
367 | run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); | |
368 | } | |
369 | ||
370 | ||
371 | my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); | |
372 | ||
373 | my $moncount = 0; | |
374 | ||
375 | my $monaddrhash = {}; | |
376 | ||
377 | foreach my $section (keys %$cfg) { | |
378 | next if $section eq 'global'; | |
379 | my $d = $cfg->{$section}; | |
380 | if ($section =~ m/^mon\./) { | |
381 | $moncount++; | |
382 | if ($d->{'mon addr'}) { | |
383 | $monaddrhash->{$d->{'mon addr'}} = $section; | |
384 | } | |
385 | } | |
386 | } | |
387 | ||
388 | my $monid; | |
389 | for (my $i = 0; $i < 7; $i++) { | |
390 | if (!$cfg->{"mon.$i"}) { | |
391 | $monid = $i; | |
392 | last; | |
393 | } | |
394 | } | |
395 | die "unable to find usable monitor id\n" if !defined($monid); | |
396 | ||
397 | my $monsection = "mon.$monid"; | |
398 | my $monaddr = PVE::Cluster::remote_node_ip($param->{node}) . ":6789"; | |
399 | my $monname = $param->{node}; | |
400 | ||
401 | die "monitor '$monsection' already exists\n" if $cfg->{$monsection}; | |
402 | die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" | |
403 | if $monaddrhash->{$monaddr}; | |
404 | ||
405 | ||
406 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; | |
407 | -d $mondir && die "monitor filesystem '$mondir' already exist\n"; | |
408 | ||
409 | my $monmap = "/tmp/monmap"; | |
410 | ||
411 | eval { | |
412 | mkdir $mondir; | |
413 | ||
414 | if ($moncount > 0) { | |
415 | my $monstat = ceph_mon_status(); # online test | |
416 | &$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]); | |
417 | } else { | |
418 | run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap"); | |
419 | } | |
420 | ||
421 | run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path"); | |
422 | }; | |
423 | my $err = $@; | |
424 | unlink $monmap; | |
425 | if ($err) { | |
426 | File::Path::remove_tree($mondir); | |
427 | die $err; | |
428 | } | |
429 | ||
430 | $cfg->{$monsection} = { | |
431 | 'host' => $monname, | |
432 | 'mon addr' => $monaddr, | |
433 | }; | |
434 | ||
435 | &$write_ceph_config($cfg); | |
436 | ||
437 | &$ceph_service_cmd('start', $monsection); | |
438 | ||
439 | return undef; | |
440 | ||
441 | }}); | |
442 | ||
443 | __PACKAGE__->register_method ({ | |
444 | name => 'destroymon', | |
445 | path => 'destroymon', | |
446 | method => 'POST', | |
447 | description => "Destroy Ceph monitor.", | |
448 | proxyto => 'node', | |
449 | protected => 1, | |
450 | parameters => { | |
451 | additionalProperties => 0, | |
452 | properties => { | |
453 | node => get_standard_option('pve-node'), | |
454 | monid => { | |
455 | description => 'Monitor ID', | |
456 | type => 'integer', | |
457 | }, | |
458 | }, | |
459 | }, | |
460 | returns => { type => 'null' }, | |
461 | code => sub { | |
462 | my ($param) = @_; | |
463 | ||
464 | &$check_ceph_inited(); | |
465 | ||
466 | my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); | |
467 | ||
468 | my $monid = $param->{monid}; | |
469 | my $monsection = "mon.$monid"; | |
470 | ||
471 | my $monstat = ceph_mon_status(); | |
472 | my $monlist = $monstat->{monmap}->{mons}; | |
473 | ||
474 | die "no such monitor id '$monid'\n" | |
475 | if !defined($cfg->{$monsection}); | |
476 | ||
477 | ||
478 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; | |
479 | -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n"; | |
480 | ||
481 | die "can't remove last monitor\n" if scalar(@$monlist) <= 1; | |
482 | ||
483 | &$run_ceph_cmd(['mon', 'remove', $monid]); | |
484 | ||
485 | eval { &$ceph_service_cmd('stop', $monsection); }; | |
486 | warn $@ if $@; | |
487 | ||
488 | delete $cfg->{$monsection}; | |
489 | &$write_ceph_config($cfg); | |
490 | File::Path::remove_tree($mondir); | |
491 | ||
492 | return undef; | |
493 | }}); | |
494 | ||
495 | __PACKAGE__->register_method ({ | |
496 | name => 'stop', | |
497 | path => 'stop', | |
498 | method => 'POST', | |
499 | description => "Stop ceph services.", | |
500 | proxyto => 'node', | |
501 | protected => 1, | |
502 | parameters => { | |
503 | additionalProperties => 0, | |
504 | properties => { | |
505 | node => get_standard_option('pve-node'), | |
506 | }, | |
507 | }, | |
508 | returns => { type => 'null' }, | |
509 | code => sub { | |
510 | my ($param) = @_; | |
511 | ||
512 | &$check_ceph_inited(); | |
513 | ||
514 | my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); | |
515 | scalar(keys %$cfg) || die "no configuration\n"; | |
516 | ||
517 | &$ceph_service_cmd('stop'); | |
518 | ||
519 | return undef; | |
520 | }}); | |
521 | ||
522 | __PACKAGE__->register_method ({ | |
523 | name => 'start', | |
524 | path => 'start', | |
525 | method => 'POST', | |
526 | description => "Start ceph services.", | |
527 | proxyto => 'node', | |
528 | protected => 1, | |
529 | parameters => { | |
530 | additionalProperties => 0, | |
531 | properties => { | |
532 | node => get_standard_option('pve-node'), | |
533 | }, | |
534 | }, | |
535 | returns => { type => 'null' }, | |
536 | code => sub { | |
537 | my ($param) = @_; | |
538 | ||
539 | &$check_ceph_inited(); | |
540 | ||
541 | my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); | |
542 | scalar(keys %$cfg) || die "no configuration\n"; | |
543 | ||
544 | &$ceph_service_cmd('start'); | |
545 | ||
546 | return undef; | |
547 | }}); | |
548 | ||
549 | __PACKAGE__->register_method ({ | |
550 | name => 'status', | |
551 | path => 'status', | |
552 | method => 'GET', | |
553 | description => "Get ceph status.", | |
554 | proxyto => 'node', | |
555 | protected => 1, | |
556 | parameters => { | |
557 | additionalProperties => 0, | |
558 | properties => { | |
559 | node => get_standard_option('pve-node'), | |
560 | }, | |
561 | }, | |
562 | returns => { type => 'object' }, | |
563 | code => sub { | |
564 | my ($param) = @_; | |
565 | ||
9662c195 | 566 | &$check_ceph_enabled(); |
38db610a | 567 | |
9662c195 | 568 | return &$run_ceph_cmd_json(['status'], quiet => 1); |
38db610a DM |
569 | }}); |
570 | ||
571 | __PACKAGE__->register_method ({ | |
572 | name => 'createosd', | |
573 | path => 'createosd', | |
574 | method => 'POST', | |
575 | description => "Create OSD", | |
576 | proxyto => 'node', | |
577 | protected => 1, | |
578 | parameters => { | |
579 | additionalProperties => 0, | |
580 | properties => { | |
581 | node => get_standard_option('pve-node'), | |
582 | dev => { | |
583 | description => "Block device name.", | |
584 | type => 'string', | |
585 | } | |
586 | }, | |
587 | }, | |
588 | returns => { type => 'null' }, | |
589 | code => sub { | |
590 | my ($param) = @_; | |
591 | ||
592 | &$check_ceph_inited(); | |
593 | ||
594 | die "not fully configured - missing '$pve_ckeyring_path'\n" | |
595 | if ! -f $pve_ckeyring_path; | |
596 | ||
597 | &$setup_pve_symlinks(); | |
598 | ||
599 | print "create OSD on $param->{dev}\n"; | |
600 | ||
601 | -b $param->{dev} || die "no such block device '$param->{dev}'\n"; | |
602 | ||
603 | my $monstat = ceph_mon_status(1); | |
604 | die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid}; | |
605 | my $fsid = $monstat->{monmap}->{fsid}; | |
606 | ||
607 | if (! -f $ceph_bootstrap_osd_keyring) { | |
608 | &$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]); | |
609 | }; | |
610 | ||
611 | run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', 'xfs', | |
612 | '--cluster', $ccname, '--cluster-uuid', $fsid, | |
613 | '--', $param->{dev}]); | |
614 | ||
615 | return undef; | |
616 | }}); | |
617 | ||
618 | __PACKAGE__->register_method ({ | |
619 | name => 'destroyosd', | |
620 | path => 'destroyosd', | |
621 | method => 'POST', | |
622 | description => "Destroy OSD", | |
623 | proxyto => 'node', | |
624 | protected => 1, | |
625 | parameters => { | |
626 | additionalProperties => 0, | |
627 | properties => { | |
628 | node => get_standard_option('pve-node'), | |
629 | osdid => { | |
630 | description => 'OSD ID', | |
631 | type => 'integer', | |
632 | }, | |
633 | }, | |
634 | }, | |
635 | returns => { type => 'null' }, | |
636 | code => sub { | |
637 | my ($param) = @_; | |
638 | ||
639 | &$check_ceph_inited(); | |
640 | ||
641 | my $osdid = $param->{osdid}; | |
642 | ||
643 | print "destroy OSD $param->{osdid}\n"; | |
644 | ||
645 | # fixme: not sure what we should do here | |
646 | ||
647 | my $stat = &$ceph_osd_status(); | |
648 | ||
649 | my $osdlist = $stat->{osds} || []; | |
650 | ||
651 | my $osdstat; | |
652 | foreach my $d (@$osdlist) { | |
653 | if ($d->{osd} == $osdid) { | |
654 | $osdstat = $d; | |
655 | last; | |
656 | } | |
657 | } | |
658 | die "no such OSD '$osdid'\n" if !$osdstat; | |
659 | ||
660 | die "osd is in use (in == 1)\n" if $osdstat->{in}; | |
661 | #&$run_ceph_cmd(['osd', 'out', $osdid]); | |
662 | ||
663 | die "osd is still runnung (up == 1)\n" if $osdstat->{up}; | |
664 | ||
665 | my $osdsection = "osd.$osdid"; | |
666 | ||
667 | eval { &$ceph_service_cmd('stop', $osdsection); }; | |
668 | warn $@ if $@; | |
669 | ||
670 | print "Remove $osdsection from the CRUSH map\n"; | |
671 | &$run_ceph_cmd(['osd', 'crush', 'remove', $osdid]); | |
672 | ||
673 | print "Remove the $osdsection authentication key.\n"; | |
674 | &$run_ceph_cmd(['auth', 'del', $osdsection]); | |
675 | ||
676 | print "Remove OSD $osdsection\n"; | |
677 | &$run_ceph_cmd(['osd', 'rm', $osdid]); | |
678 | ||
679 | return undef; | |
680 | }}); |