]>
Commit | Line | Data |
---|---|---|
38db610a DM |
1 | package PVE::API2::Ceph; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | use File::Basename; | |
6 | use File::Path; | |
7 | use POSIX qw (LONG_MAX); | |
13f4d762 DM |
8 | use Cwd qw(abs_path); |
9 | use IO::Dir; | |
eac465a5 | 10 | use UUID; |
f7e342ea | 11 | use Net::IP; |
38db610a DM |
12 | |
13 | use PVE::SafeSyslog; | |
13f4d762 | 14 | use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach); |
38db610a DM |
15 | use PVE::Exception qw(raise raise_param_exc); |
16 | use PVE::INotify; | |
17 | use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file); | |
18 | use PVE::AccessControl; | |
19 | use PVE::Storage; | |
20 | use PVE::RESTHandler; | |
21 | use PVE::RPCEnvironment; | |
22 | use PVE::JSONSchema qw(get_standard_option); | |
23 | use JSON; | |
970236b3 | 24 | use PVE::RADOS; |
a34866f0 | 25 | use PVE::CephTools; |
38db610a DM |
26 | |
27 | use base qw(PVE::RESTHandler); | |
28 | ||
29 | use Data::Dumper; # fixme: remove | |
30 | ||
0e5816e4 | 31 | |
2f804640 DM |
32 | # we can use longer rados timeout when inside workers |
33 | my $long_rados_timeout = 60; | |
34 | ||
a34866f0 | 35 | my $pve_osd_default_journal_size = 1024*5; |
38db610a | 36 | |
38db610a | 37 | |
13f4d762 DM |
38 | sub list_disks { |
39 | my $disklist = {}; | |
40 | ||
41 | my $fd = IO::File->new("/proc/mounts", "r") || | |
42 | die "unable to open /proc/mounts - $!\n"; | |
43 | ||
44 | my $mounted = {}; | |
45 | ||
46 | while (defined(my $line = <$fd>)) { | |
47 | my ($dev, $path, $fstype) = split(/\s+/, $line); | |
48 | next if !($dev && $path && $fstype); | |
49 | next if $dev !~ m|^/dev/|; | |
50 | my $real_dev = abs_path($dev); | |
51 | $mounted->{$real_dev} = $path; | |
52 | } | |
53 | close($fd); | |
54 | ||
55 | my $dev_is_mounted = sub { | |
56 | my ($dev) = @_; | |
57 | return $mounted->{$dev}; | |
58 | }; | |
59 | ||
60 | my $dir_is_epmty = sub { | |
61 | my ($dir) = @_; | |
62 | ||
63 | my $dh = IO::Dir->new ($dir); | |
64 | return 1 if !$dh; | |
65 | ||
66 | while (defined(my $tmp = $dh->read)) { | |
67 | next if $tmp eq '.' || $tmp eq '..'; | |
68 | $dh->close; | |
69 | return 0; | |
70 | } | |
71 | $dh->close; | |
72 | return 1; | |
73 | }; | |
74 | ||
0e5816e4 DM |
75 | my $journal_uuid = '45b0969e-9b03-4f30-b4c6-b4b80ceff106'; |
76 | ||
77 | my $journalhash = {}; | |
78 | dir_glob_foreach('/dev/disk/by-parttypeuuid', "$journal_uuid\..+", sub { | |
79 | my ($entry) = @_; | |
80 | my $real_dev = abs_path("/dev/disk/by-parttypeuuid/$entry"); | |
81 | $journalhash->{$real_dev} = 1; | |
82 | }); | |
83 | ||
13f4d762 DM |
84 | dir_glob_foreach('/sys/block', '.*', sub { |
85 | my ($dev) = @_; | |
86 | ||
87 | return if $dev eq '.'; | |
88 | return if $dev eq '..'; | |
89 | ||
90 | return if $dev =~ m|^ram\d+$|; # skip ram devices | |
91 | return if $dev =~ m|^loop\d+$|; # skip loop devices | |
92 | return if $dev =~ m|^md\d+$|; # skip md devices | |
93 | return if $dev =~ m|^dm-.*$|; # skip dm related things | |
94 | return if $dev =~ m|^fd\d+$|; # skip Floppy | |
95 | return if $dev =~ m|^sr\d+$|; # skip CDs | |
96 | ||
97 | my $devdir = "/sys/block/$dev/device"; | |
98 | return if ! -d $devdir; | |
99 | ||
100 | my $size = file_read_firstline("/sys/block/$dev/size"); | |
101 | return if !$size; | |
102 | ||
103 | $size = $size * 512; | |
104 | ||
105 | my $info = `udevadm info --path /sys/block/$dev --query all`; | |
106 | return if !$info; | |
107 | ||
108 | return if $info !~ m/^E: DEVTYPE=disk$/m; | |
109 | return if $info =~ m/^E: ID_CDROM/m; | |
110 | ||
111 | my $serial = 'unknown'; | |
112 | if ($info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m) { | |
113 | $serial = $1; | |
114 | } | |
115 | ||
0e5816e4 DM |
116 | my $gpt = 0; |
117 | if ($info =~ m/^E: ID_PART_TABLE_TYPE=gpt$/m) { | |
118 | $gpt = 1; | |
119 | } | |
120 | ||
121 | # detect SSD (fixme - currently only works for ATA disks) | |
122 | my $rpm = 7200; # default guess | |
123 | if ($info =~ m/^E: ID_ATA_ROTATION_RATE_RPM=(\d+)$/m) { | |
124 | $rpm = $1; | |
125 | } | |
126 | ||
13f4d762 DM |
127 | my $vendor = file_read_firstline("$devdir/vendor") || 'unknown'; |
128 | my $model = file_read_firstline("$devdir/model") || 'unknown'; | |
129 | ||
0e5816e4 DM |
130 | my $used; |
131 | ||
132 | $used = 'LVM' if !&$dir_is_epmty("/sys/block/$dev/holders"); | |
13f4d762 | 133 | |
0e5816e4 | 134 | $used = 'mounted' if &$dev_is_mounted("/dev/$dev"); |
13f4d762 DM |
135 | |
136 | $disklist->{$dev} = { | |
137 | vendor => $vendor, | |
138 | model => $model, | |
139 | size => $size, | |
140 | serial => $serial, | |
0e5816e4 DM |
141 | gpt => $gpt, |
142 | rmp => $rpm, | |
13f4d762 DM |
143 | }; |
144 | ||
145 | my $osdid = -1; | |
146 | ||
0e5816e4 DM |
147 | my $journal_count = 0; |
148 | ||
a7a7fb00 DM |
149 | my $found_partitions; |
150 | my $found_lvm; | |
151 | my $found_mountpoints; | |
13f4d762 DM |
152 | dir_glob_foreach("/sys/block/$dev", "$dev.+", sub { |
153 | my ($part) = @_; | |
a7a7fb00 DM |
154 | |
155 | $found_partitions = 1; | |
156 | ||
13f4d762 | 157 | if (my $mp = &$dev_is_mounted("/dev/$part")) { |
a7a7fb00 | 158 | $found_mountpoints = 1; |
13f4d762 DM |
159 | if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) { |
160 | $osdid = $1; | |
161 | } | |
0e5816e4 DM |
162 | } |
163 | if (!&$dir_is_epmty("/sys/block/$dev/$part/holders")) { | |
a7a7fb00 | 164 | $found_lvm = 1; |
0e5816e4 | 165 | } |
0e5816e4 | 166 | $journal_count++ if $journalhash->{"/dev/$part"}; |
13f4d762 DM |
167 | }); |
168 | ||
a7a7fb00 DM |
169 | $used = 'mounted' if $found_mountpoints && !$used; |
170 | $used = 'LVM' if $found_lvm && !$used; | |
171 | $used = 'partitions' if $found_partitions && !$used; | |
172 | ||
0e5816e4 | 173 | $disklist->{$dev}->{used} = $used if $used; |
13f4d762 | 174 | $disklist->{$dev}->{osdid} = $osdid; |
0e5816e4 DM |
175 | $disklist->{$dev}->{journals} = $journal_count; |
176 | }); | |
13f4d762 DM |
177 | |
178 | return $disklist; | |
179 | } | |
180 | ||
0e5816e4 DM |
181 | my $lookup_diskinfo = sub { |
182 | my ($disklist, $disk) = @_; | |
183 | ||
184 | my $real_dev = abs_path($disk); | |
185 | $real_dev =~ s|/dev/||; | |
186 | my $diskinfo = $disklist->{$real_dev}; | |
187 | ||
188 | die "disk '$disk' not found in disk list\n" if !$diskinfo; | |
189 | ||
190 | return wantarray ? ($diskinfo, $real_dev) : $diskinfo; | |
191 | }; | |
192 | ||
193 | ||
194 | my $count_journal_disks = sub { | |
195 | my ($disklist, $disk) = @_; | |
196 | ||
197 | my $count = 0; | |
198 | ||
199 | my ($diskinfo, $real_dev) = &$lookup_diskinfo($disklist, $disk); | |
200 | die "journal disk '$disk' does not contain a GUID partition table\n" | |
201 | if !$diskinfo->{gpt}; | |
202 | ||
203 | $count = $diskinfo->{journals} if $diskinfo->{journals}; | |
204 | ||
205 | return $count; | |
206 | }; | |
207 | ||
38db610a DM |
208 | __PACKAGE__->register_method ({ |
209 | name => 'index', | |
210 | path => '', | |
211 | method => 'GET', | |
212 | description => "Directory index.", | |
213 | permissions => { user => 'all' }, | |
214 | parameters => { | |
215 | additionalProperties => 0, | |
216 | properties => { | |
217 | node => get_standard_option('pve-node'), | |
218 | }, | |
219 | }, | |
220 | returns => { | |
221 | type => 'array', | |
222 | items => { | |
223 | type => "object", | |
224 | properties => {}, | |
225 | }, | |
226 | links => [ { rel => 'child', href => "{name}" } ], | |
227 | }, | |
228 | code => sub { | |
229 | my ($param) = @_; | |
230 | ||
231 | my $result = [ | |
232 | { name => 'init' }, | |
39e1ad70 DM |
233 | { name => 'mon' }, |
234 | { name => 'osd' }, | |
b0537f7b | 235 | { name => 'pools' }, |
38db610a DM |
236 | { name => 'stop' }, |
237 | { name => 'start' }, | |
238 | { name => 'status' }, | |
2f692121 | 239 | { name => 'crush' }, |
68e0c4bd | 240 | { name => 'config' }, |
570278fa | 241 | { name => 'log' }, |
13f4d762 | 242 | { name => 'disks' }, |
38db610a DM |
243 | ]; |
244 | ||
245 | return $result; | |
246 | }}); | |
247 | ||
13f4d762 DM |
248 | __PACKAGE__->register_method ({ |
249 | name => 'disks', | |
250 | path => 'disks', | |
251 | method => 'GET', | |
252 | description => "List local disks.", | |
253 | proxyto => 'node', | |
254 | protected => 1, | |
255 | parameters => { | |
256 | additionalProperties => 0, | |
257 | properties => { | |
258 | node => get_standard_option('pve-node'), | |
0e5816e4 DM |
259 | type => { |
260 | description => "Only list specific types of disks.", | |
261 | type => 'string', | |
262 | enum => ['unused', 'journal_disks'], | |
263 | optional => 1, | |
264 | }, | |
13f4d762 DM |
265 | }, |
266 | }, | |
267 | returns => { | |
268 | type => 'array', | |
269 | items => { | |
270 | type => "object", | |
271 | properties => { | |
272 | dev => { type => 'string' }, | |
0e5816e4 DM |
273 | used => { type => 'string', optional => 1 }, |
274 | gpt => { type => 'boolean' }, | |
13f4d762 DM |
275 | size => { type => 'integer' }, |
276 | osdid => { type => 'integer' }, | |
277 | vendor => { type => 'string', optional => 1 }, | |
278 | model => { type => 'string', optional => 1 }, | |
279 | serial => { type => 'string', optional => 1 }, | |
280 | }, | |
281 | }, | |
282 | # links => [ { rel => 'child', href => "{}" } ], | |
283 | }, | |
284 | code => sub { | |
285 | my ($param) = @_; | |
286 | ||
a34866f0 | 287 | PVE::CephTools::check_ceph_inited(); |
13f4d762 | 288 | |
0e5816e4 | 289 | my $disks = list_disks(); |
13f4d762 | 290 | |
0e5816e4 DM |
291 | my $res = []; |
292 | foreach my $dev (keys %$disks) { | |
293 | my $d = $disks->{$dev}; | |
294 | if ($param->{type}) { | |
295 | if ($param->{type} eq 'journal_disks') { | |
296 | next if $d->{osdid} >= 0; | |
297 | next if !$d->{gpt}; | |
298 | } elsif ($param->{type} eq 'unused') { | |
299 | next if $d->{used}; | |
300 | } else { | |
301 | die "internal error"; # should not happen | |
302 | } | |
303 | } | |
304 | ||
305 | $d->{dev} = "/dev/$dev"; | |
306 | push @$res, $d; | |
307 | } | |
308 | ||
309 | return $res; | |
13f4d762 DM |
310 | }}); |
311 | ||
68e0c4bd DM |
312 | __PACKAGE__->register_method ({ |
313 | name => 'config', | |
314 | path => 'config', | |
315 | method => 'GET', | |
316 | description => "Get Ceph configuration.", | |
317 | parameters => { | |
318 | additionalProperties => 0, | |
319 | properties => { | |
320 | node => get_standard_option('pve-node'), | |
321 | }, | |
322 | }, | |
323 | returns => { type => 'string' }, | |
324 | code => sub { | |
325 | my ($param) = @_; | |
326 | ||
a34866f0 | 327 | PVE::CephTools::check_ceph_inited(); |
68e0c4bd | 328 | |
a34866f0 DM |
329 | my $path = PVE::CephTools::get_config('pve_ceph_cfgpath'); |
330 | return PVE::Tools::file_get_contents($path); | |
68e0c4bd DM |
331 | |
332 | }}); | |
333 | ||
334 | __PACKAGE__->register_method ({ | |
335 | name => 'listmon', | |
39e1ad70 | 336 | path => 'mon', |
68e0c4bd DM |
337 | method => 'GET', |
338 | description => "Get Ceph monitor list.", | |
339 | proxyto => 'node', | |
340 | protected => 1, | |
341 | parameters => { | |
342 | additionalProperties => 0, | |
343 | properties => { | |
344 | node => get_standard_option('pve-node'), | |
345 | }, | |
346 | }, | |
347 | returns => { | |
348 | type => 'array', | |
349 | items => { | |
350 | type => "object", | |
351 | properties => { | |
352 | name => { type => 'string' }, | |
353 | addr => { type => 'string' }, | |
354 | }, | |
355 | }, | |
39e1ad70 | 356 | links => [ { rel => 'child', href => "{name}" } ], |
68e0c4bd DM |
357 | }, |
358 | code => sub { | |
359 | my ($param) = @_; | |
360 | ||
a34866f0 | 361 | PVE::CephTools::check_ceph_inited(); |
68e0c4bd DM |
362 | |
363 | my $res = []; | |
364 | ||
a34866f0 | 365 | my $cfg = PVE::CephTools::parse_ceph_config(); |
68e0c4bd DM |
366 | |
367 | my $monhash = {}; | |
368 | foreach my $section (keys %$cfg) { | |
369 | my $d = $cfg->{$section}; | |
370 | if ($section =~ m/^mon\.(\S+)$/) { | |
371 | my $monid = $1; | |
372 | if ($d->{'mon addr'} && $d->{'host'}) { | |
373 | $monhash->{$monid} = { | |
374 | addr => $d->{'mon addr'}, | |
375 | host => $d->{'host'}, | |
376 | name => $monid, | |
377 | } | |
378 | } | |
379 | } | |
380 | } | |
381 | ||
382 | eval { | |
36fd0190 | 383 | my $rados = PVE::RADOS->new(); |
970236b3 | 384 | my $monstat = $rados->mon_command({ prefix => 'mon_status' }); |
68e0c4bd DM |
385 | my $mons = $monstat->{monmap}->{mons}; |
386 | foreach my $d (@$mons) { | |
387 | next if !defined($d->{name}); | |
388 | $monhash->{$d->{name}}->{rank} = $d->{rank}; | |
389 | $monhash->{$d->{name}}->{addr} = $d->{addr}; | |
390 | if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) { | |
391 | $monhash->{$d->{name}}->{quorum} = 1; | |
392 | } | |
393 | } | |
394 | }; | |
395 | warn $@ if $@; | |
396 | ||
397 | return PVE::RESTHandler::hash_to_array($monhash, 'name'); | |
398 | }}); | |
399 | ||
38db610a DM |
400 | __PACKAGE__->register_method ({ |
401 | name => 'init', | |
402 | path => 'init', | |
403 | method => 'POST', | |
a08988dc | 404 | description => "Create initial ceph default configuration and setup symlinks.", |
38db610a DM |
405 | proxyto => 'node', |
406 | protected => 1, | |
407 | parameters => { | |
408 | additionalProperties => 0, | |
409 | properties => { | |
410 | node => get_standard_option('pve-node'), | |
f7e342ea DM |
411 | network => { |
412 | description => "Use specific network for all ceph related traffic", | |
413 | type => 'string', format => 'CIDR', | |
414 | optional => 1, | |
415 | maxLength => 128, | |
416 | }, | |
38db610a DM |
417 | size => { |
418 | description => 'Number of replicas per object', | |
419 | type => 'integer', | |
420 | default => 2, | |
421 | optional => 1, | |
422 | minimum => 1, | |
423 | maximum => 3, | |
424 | }, | |
425 | pg_bits => { | |
426 | description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)", | |
427 | type => 'integer', | |
a08988dc | 428 | default => 6, |
38db610a DM |
429 | optional => 1, |
430 | minimum => 6, | |
431 | maximum => 14, | |
432 | }, | |
433 | }, | |
434 | }, | |
435 | returns => { type => 'null' }, | |
436 | code => sub { | |
437 | my ($param) = @_; | |
438 | ||
a34866f0 | 439 | PVE::CephTools::check_ceph_installed(); |
38db610a | 440 | |
a08988dc | 441 | # simply load old config if it already exists |
a34866f0 | 442 | my $cfg = PVE::CephTools::parse_ceph_config(); |
38db610a | 443 | |
a08988dc DM |
444 | if (!$cfg->{global}) { |
445 | ||
446 | my $fsid; | |
447 | my $uuid; | |
38db610a | 448 | |
a08988dc DM |
449 | UUID::generate($uuid); |
450 | UUID::unparse($uuid, $fsid); | |
451 | ||
452 | $cfg->{global} = { | |
453 | 'fsid' => $fsid, | |
454 | 'auth supported' => 'cephx', | |
455 | 'auth cluster required' => 'cephx', | |
456 | 'auth service required' => 'cephx', | |
457 | 'auth client required' => 'cephx', | |
458 | 'filestore xattr use omap' => 'true', | |
0e5816e4 | 459 | 'osd journal size' => $pve_osd_default_journal_size, |
a08988dc DM |
460 | 'osd pool default min size' => 1, |
461 | }; | |
462 | ||
463 | # this does not work for default pools | |
464 | #'osd pool default pg num' => $pg_num, | |
465 | #'osd pool default pgp num' => $pg_num, | |
466 | } | |
467 | ||
88a6e29a | 468 | $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring'; |
52d7be41 | 469 | $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring'; |
38db610a | 470 | |
a08988dc | 471 | $cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size}; |
0e5816e4 | 472 | |
a08988dc DM |
473 | if ($param->{pg_bits}) { |
474 | $cfg->{global}->{'osd pg bits'} = $param->{pg_bits}; | |
475 | $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits}; | |
476 | } | |
f7e342ea DM |
477 | |
478 | if ($param->{network}) { | |
479 | $cfg->{global}->{'public network'} = $param->{network}; | |
480 | $cfg->{global}->{'cluster network'} = $param->{network}; | |
481 | } | |
482 | ||
a34866f0 | 483 | PVE::CephTools::write_ceph_config($cfg); |
38db610a | 484 | |
a34866f0 | 485 | PVE::CephTools::setup_pve_symlinks(); |
38db610a DM |
486 | |
487 | return undef; | |
488 | }}); | |
489 | ||
f7e342ea DM |
490 | my $find_node_ip = sub { |
491 | my ($cidr) = @_; | |
492 | ||
493 | my $config = PVE::INotify::read_file('interfaces'); | |
494 | ||
495 | my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n"; | |
496 | ||
497 | foreach my $iface (keys %$config) { | |
498 | my $d = $config->{$iface}; | |
499 | next if !$d->{address}; | |
500 | my $a = Net::IP->new($d->{address}); | |
501 | next if !$a; | |
502 | return $d->{address} if $net->overlaps($a); | |
503 | } | |
504 | ||
505 | die "unable to find local address within network '$cidr'\n"; | |
506 | }; | |
507 | ||
38db610a DM |
508 | __PACKAGE__->register_method ({ |
509 | name => 'createmon', | |
39e1ad70 | 510 | path => 'mon', |
38db610a DM |
511 | method => 'POST', |
512 | description => "Create Ceph Monitor", | |
513 | proxyto => 'node', | |
514 | protected => 1, | |
515 | parameters => { | |
516 | additionalProperties => 0, | |
517 | properties => { | |
518 | node => get_standard_option('pve-node'), | |
519 | }, | |
520 | }, | |
52d7be41 | 521 | returns => { type => 'string' }, |
38db610a DM |
522 | code => sub { |
523 | my ($param) = @_; | |
524 | ||
a34866f0 | 525 | PVE::CephTools::check_ceph_inited(); |
38db610a | 526 | |
a34866f0 | 527 | PVE::CephTools::setup_pve_symlinks(); |
38db610a | 528 | |
52d7be41 | 529 | my $rpcenv = PVE::RPCEnvironment::get(); |
38db610a | 530 | |
52d7be41 | 531 | my $authuser = $rpcenv->get_user(); |
38db610a | 532 | |
a34866f0 | 533 | my $cfg = PVE::CephTools::parse_ceph_config(); |
38db610a DM |
534 | |
535 | my $moncount = 0; | |
536 | ||
537 | my $monaddrhash = {}; | |
538 | ||
539 | foreach my $section (keys %$cfg) { | |
540 | next if $section eq 'global'; | |
541 | my $d = $cfg->{$section}; | |
542 | if ($section =~ m/^mon\./) { | |
543 | $moncount++; | |
544 | if ($d->{'mon addr'}) { | |
545 | $monaddrhash->{$d->{'mon addr'}} = $section; | |
546 | } | |
547 | } | |
548 | } | |
549 | ||
550 | my $monid; | |
551 | for (my $i = 0; $i < 7; $i++) { | |
552 | if (!$cfg->{"mon.$i"}) { | |
553 | $monid = $i; | |
554 | last; | |
555 | } | |
556 | } | |
557 | die "unable to find usable monitor id\n" if !defined($monid); | |
558 | ||
f7e342ea DM |
559 | my $monsection = "mon.$monid"; |
560 | my $ip; | |
561 | if (my $pubnet = $cfg->{global}->{'public network'}) { | |
562 | $ip = &$find_node_ip($pubnet); | |
563 | } else { | |
564 | $ip = PVE::Cluster::remote_node_ip($param->{node}); | |
565 | } | |
566 | ||
567 | my $monaddr = "$ip:6789"; | |
38db610a DM |
568 | my $monname = $param->{node}; |
569 | ||
570 | die "monitor '$monsection' already exists\n" if $cfg->{$monsection}; | |
571 | die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" | |
572 | if $monaddrhash->{$monaddr}; | |
573 | ||
52d7be41 DM |
574 | my $worker = sub { |
575 | my $upid = shift; | |
38db610a | 576 | |
a34866f0 DM |
577 | my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path'); |
578 | ||
52d7be41 DM |
579 | if (! -f $pve_ckeyring_path) { |
580 | run_command("ceph-authtool $pve_ckeyring_path --create-keyring " . | |
581 | "--gen-key -n client.admin"); | |
582 | } | |
38db610a | 583 | |
a34866f0 | 584 | my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path'); |
52d7be41 DM |
585 | if (! -f $pve_mon_key_path) { |
586 | run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); | |
587 | run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " . | |
588 | "--cap mds 'allow' " . | |
589 | "--cap osd 'allow *' " . | |
590 | "--cap mon 'allow *'"); | |
591 | run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); | |
592 | run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); | |
38db610a DM |
593 | } |
594 | ||
a34866f0 DM |
595 | my $ccname = PVE::CephTools::get_config('ccname'); |
596 | ||
52d7be41 DM |
597 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; |
598 | -d $mondir && die "monitor filesystem '$mondir' already exist\n"; | |
599 | ||
600 | my $monmap = "/tmp/monmap"; | |
601 | ||
602 | eval { | |
603 | mkdir $mondir; | |
604 | ||
605 | if ($moncount > 0) { | |
2f804640 | 606 | my $rados = PVE::RADOS->new(timeout => $long_rados_timeout); |
970236b3 DM |
607 | my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' }); |
608 | PVE::Tools::file_set_contents($monmap, $mapdata); | |
52d7be41 DM |
609 | } else { |
610 | run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap"); | |
611 | } | |
38db610a | 612 | |
52d7be41 DM |
613 | run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path"); |
614 | }; | |
615 | my $err = $@; | |
616 | unlink $monmap; | |
617 | if ($err) { | |
618 | File::Path::remove_tree($mondir); | |
619 | die $err; | |
620 | } | |
38db610a | 621 | |
52d7be41 DM |
622 | $cfg->{$monsection} = { |
623 | 'host' => $monname, | |
624 | 'mon addr' => $monaddr, | |
625 | }; | |
38db610a | 626 | |
a34866f0 | 627 | PVE::CephTools::write_ceph_config($cfg); |
38db610a | 628 | |
a34866f0 | 629 | PVE::CephTools::ceph_service_cmd('start', $monsection); |
52d7be41 | 630 | }; |
38db610a | 631 | |
52d7be41 | 632 | return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker); |
38db610a DM |
633 | }}); |
634 | ||
635 | __PACKAGE__->register_method ({ | |
636 | name => 'destroymon', | |
39e1ad70 DM |
637 | path => 'mon/{monid}', |
638 | method => 'DELETE', | |
38db610a DM |
639 | description => "Destroy Ceph monitor.", |
640 | proxyto => 'node', | |
641 | protected => 1, | |
642 | parameters => { | |
643 | additionalProperties => 0, | |
644 | properties => { | |
645 | node => get_standard_option('pve-node'), | |
646 | monid => { | |
647 | description => 'Monitor ID', | |
648 | type => 'integer', | |
649 | }, | |
650 | }, | |
651 | }, | |
52d7be41 | 652 | returns => { type => 'string' }, |
38db610a DM |
653 | code => sub { |
654 | my ($param) = @_; | |
655 | ||
52d7be41 DM |
656 | my $rpcenv = PVE::RPCEnvironment::get(); |
657 | ||
658 | my $authuser = $rpcenv->get_user(); | |
659 | ||
a34866f0 | 660 | PVE::CephTools::check_ceph_inited(); |
38db610a | 661 | |
a34866f0 | 662 | my $cfg = PVE::CephTools::parse_ceph_config(); |
38db610a DM |
663 | |
664 | my $monid = $param->{monid}; | |
665 | my $monsection = "mon.$monid"; | |
666 | ||
36fd0190 | 667 | my $rados = PVE::RADOS->new(); |
970236b3 | 668 | my $monstat = $rados->mon_command({ prefix => 'mon_status' }); |
38db610a DM |
669 | my $monlist = $monstat->{monmap}->{mons}; |
670 | ||
671 | die "no such monitor id '$monid'\n" | |
672 | if !defined($cfg->{$monsection}); | |
673 | ||
a34866f0 DM |
674 | my $ccname = PVE::CephTools::get_config('ccname'); |
675 | ||
38db610a DM |
676 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; |
677 | -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n"; | |
678 | ||
679 | die "can't remove last monitor\n" if scalar(@$monlist) <= 1; | |
680 | ||
52d7be41 DM |
681 | my $worker = sub { |
682 | my $upid = shift; | |
38db610a | 683 | |
2f804640 DM |
684 | # reopen with longer timeout |
685 | $rados = PVE::RADOS->new(timeout => $long_rados_timeout); | |
f26b46db | 686 | |
6e3c2f47 | 687 | $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' }); |
38db610a | 688 | |
a34866f0 | 689 | eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); }; |
52d7be41 | 690 | warn $@ if $@; |
38db610a | 691 | |
52d7be41 | 692 | delete $cfg->{$monsection}; |
a34866f0 | 693 | PVE::CephTools::write_ceph_config($cfg); |
52d7be41 DM |
694 | File::Path::remove_tree($mondir); |
695 | }; | |
696 | ||
697 | return $rpcenv->fork_worker('cephdestroymon', $monsection, $authuser, $worker); | |
38db610a DM |
698 | }}); |
699 | ||
700 | __PACKAGE__->register_method ({ | |
701 | name => 'stop', | |
702 | path => 'stop', | |
703 | method => 'POST', | |
704 | description => "Stop ceph services.", | |
705 | proxyto => 'node', | |
706 | protected => 1, | |
707 | parameters => { | |
708 | additionalProperties => 0, | |
709 | properties => { | |
710 | node => get_standard_option('pve-node'), | |
68e0c4bd DM |
711 | service => { |
712 | description => 'Ceph service name.', | |
713 | type => 'string', | |
714 | optional => 1, | |
715 | pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}', | |
716 | }, | |
38db610a DM |
717 | }, |
718 | }, | |
68e0c4bd | 719 | returns => { type => 'string' }, |
38db610a DM |
720 | code => sub { |
721 | my ($param) = @_; | |
722 | ||
68e0c4bd DM |
723 | my $rpcenv = PVE::RPCEnvironment::get(); |
724 | ||
725 | my $authuser = $rpcenv->get_user(); | |
726 | ||
a34866f0 | 727 | PVE::CephTools::check_ceph_inited(); |
38db610a | 728 | |
a34866f0 | 729 | my $cfg = PVE::CephTools::parse_ceph_config(); |
38db610a DM |
730 | scalar(keys %$cfg) || die "no configuration\n"; |
731 | ||
68e0c4bd DM |
732 | my $worker = sub { |
733 | my $upid = shift; | |
38db610a | 734 | |
68e0c4bd DM |
735 | my $cmd = ['stop']; |
736 | if ($param->{service}) { | |
737 | push @$cmd, $param->{service}; | |
738 | } | |
739 | ||
a34866f0 | 740 | PVE::CephTools::ceph_service_cmd(@$cmd); |
68e0c4bd DM |
741 | }; |
742 | ||
743 | return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph', | |
744 | $authuser, $worker); | |
38db610a DM |
745 | }}); |
746 | ||
747 | __PACKAGE__->register_method ({ | |
748 | name => 'start', | |
749 | path => 'start', | |
750 | method => 'POST', | |
751 | description => "Start ceph services.", | |
752 | proxyto => 'node', | |
753 | protected => 1, | |
754 | parameters => { | |
755 | additionalProperties => 0, | |
756 | properties => { | |
757 | node => get_standard_option('pve-node'), | |
68e0c4bd DM |
758 | service => { |
759 | description => 'Ceph service name.', | |
760 | type => 'string', | |
761 | optional => 1, | |
762 | pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}', | |
763 | }, | |
38db610a DM |
764 | }, |
765 | }, | |
68e0c4bd | 766 | returns => { type => 'string' }, |
38db610a DM |
767 | code => sub { |
768 | my ($param) = @_; | |
769 | ||
68e0c4bd DM |
770 | my $rpcenv = PVE::RPCEnvironment::get(); |
771 | ||
772 | my $authuser = $rpcenv->get_user(); | |
773 | ||
a34866f0 | 774 | PVE::CephTools::check_ceph_inited(); |
38db610a | 775 | |
a34866f0 | 776 | my $cfg = PVE::CephTools::parse_ceph_config(); |
38db610a DM |
777 | scalar(keys %$cfg) || die "no configuration\n"; |
778 | ||
68e0c4bd DM |
779 | my $worker = sub { |
780 | my $upid = shift; | |
38db610a | 781 | |
68e0c4bd DM |
782 | my $cmd = ['start']; |
783 | if ($param->{service}) { | |
784 | push @$cmd, $param->{service}; | |
785 | } | |
786 | ||
a34866f0 | 787 | PVE::CephTools::ceph_service_cmd(@$cmd); |
68e0c4bd DM |
788 | }; |
789 | ||
790 | return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph', | |
791 | $authuser, $worker); | |
38db610a DM |
792 | }}); |
793 | ||
794 | __PACKAGE__->register_method ({ | |
795 | name => 'status', | |
796 | path => 'status', | |
797 | method => 'GET', | |
798 | description => "Get ceph status.", | |
799 | proxyto => 'node', | |
800 | protected => 1, | |
801 | parameters => { | |
802 | additionalProperties => 0, | |
803 | properties => { | |
804 | node => get_standard_option('pve-node'), | |
805 | }, | |
806 | }, | |
807 | returns => { type => 'object' }, | |
808 | code => sub { | |
809 | my ($param) = @_; | |
810 | ||
a34866f0 | 811 | PVE::CephTools::check_ceph_enabled(); |
38db610a | 812 | |
36fd0190 | 813 | my $rados = PVE::RADOS->new(); |
970236b3 | 814 | return $rados->mon_command({ prefix => 'status' }); |
38db610a DM |
815 | }}); |
816 | ||
b0537f7b DM |
817 | __PACKAGE__->register_method ({ |
818 | name => 'lspools', | |
819 | path => 'pools', | |
820 | method => 'GET', | |
821 | description => "List all pools.", | |
822 | proxyto => 'node', | |
823 | protected => 1, | |
824 | parameters => { | |
825 | additionalProperties => 0, | |
826 | properties => { | |
827 | node => get_standard_option('pve-node'), | |
828 | }, | |
829 | }, | |
830 | returns => { | |
831 | type => 'array', | |
832 | items => { | |
833 | type => "object", | |
834 | properties => { | |
835 | pool => { type => 'integer' }, | |
836 | pool_name => { type => 'string' }, | |
837 | size => { type => 'integer' }, | |
838 | }, | |
839 | }, | |
840 | links => [ { rel => 'child', href => "{pool_name}" } ], | |
841 | }, | |
842 | code => sub { | |
843 | my ($param) = @_; | |
844 | ||
a34866f0 | 845 | PVE::CephTools::check_ceph_inited(); |
b0537f7b | 846 | |
36fd0190 | 847 | my $rados = PVE::RADOS->new(); |
970236b3 | 848 | my $res = $rados->mon_command({ prefix => 'osd dump' }); |
b0537f7b DM |
849 | |
850 | my $data = []; | |
851 | foreach my $e (@{$res->{pools}}) { | |
852 | my $d = {}; | |
853 | foreach my $attr (qw(pool pool_name size min_size pg_num crush_ruleset)) { | |
854 | $d->{$attr} = $e->{$attr} if defined($e->{$attr}); | |
855 | } | |
856 | push @$data, $d; | |
857 | } | |
858 | ||
859 | return $data; | |
860 | }}); | |
861 | ||
862 | __PACKAGE__->register_method ({ | |
863 | name => 'createpool', | |
864 | path => 'pools', | |
865 | method => 'POST', | |
866 | description => "Create POOL", | |
867 | proxyto => 'node', | |
868 | protected => 1, | |
869 | parameters => { | |
870 | additionalProperties => 0, | |
871 | properties => { | |
872 | node => get_standard_option('pve-node'), | |
873 | name => { | |
874 | description => "The name of the pool. It must be unique.", | |
875 | type => 'string', | |
876 | }, | |
c7881bf6 DM |
877 | size => { |
878 | description => 'Number of replicas per object', | |
879 | type => 'integer', | |
880 | default => 2, | |
881 | optional => 1, | |
882 | minimum => 1, | |
883 | maximum => 3, | |
884 | }, | |
74089559 DM |
885 | min_size => { |
886 | description => 'Minimum number of replicas per object', | |
887 | type => 'integer', | |
888 | default => 1, | |
889 | optional => 1, | |
890 | minimum => 1, | |
891 | maximum => 3, | |
892 | }, | |
b0537f7b DM |
893 | pg_num => { |
894 | description => "Number of placement groups.", | |
895 | type => 'integer', | |
d6261e31 | 896 | default => 64, |
b0537f7b DM |
897 | optional => 1, |
898 | minimum => 8, | |
899 | maximum => 32768, | |
946662f6 DM |
900 | }, |
901 | crush_ruleset => { | |
902 | description => "The ruleset to use for mapping object placement in the cluster.", | |
903 | type => 'integer', | |
904 | minimum => 0, | |
905 | maximum => 32768, | |
906 | default => 0, | |
907 | optional => 1, | |
908 | }, | |
b0537f7b DM |
909 | }, |
910 | }, | |
911 | returns => { type => 'null' }, | |
912 | code => sub { | |
913 | my ($param) = @_; | |
914 | ||
a34866f0 DM |
915 | PVE::CephTools::check_ceph_inited(); |
916 | ||
917 | my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path'); | |
b0537f7b DM |
918 | |
919 | die "not fully configured - missing '$pve_ckeyring_path'\n" | |
920 | if ! -f $pve_ckeyring_path; | |
921 | ||
d6261e31 | 922 | my $pg_num = $param->{pg_num} || 64; |
c7881bf6 | 923 | my $size = $param->{size} || 2; |
74089559 | 924 | my $min_size = $param->{min_size} || 1; |
970236b3 | 925 | my $ruleset = $param->{crush_ruleset} || 0; |
36fd0190 | 926 | my $rados = PVE::RADOS->new(); |
970236b3 DM |
927 | |
928 | $rados->mon_command({ | |
929 | prefix => "osd pool create", | |
930 | pool => $param->{name}, | |
931 | pg_num => int($pg_num), | |
932 | # this does not work for unknown reason | |
933 | # properties => ["size=$size", "min_size=$min_size", "crush_ruleset=$ruleset"], | |
6e3c2f47 | 934 | format => 'plain', |
970236b3 | 935 | }); |
b0537f7b | 936 | |
970236b3 DM |
937 | $rados->mon_command({ |
938 | prefix => "osd pool set", | |
939 | pool => $param->{name}, | |
940 | var => 'min_size', | |
6e3c2f47 DM |
941 | val => $min_size, |
942 | format => 'plain', | |
970236b3 | 943 | }); |
74089559 | 944 | |
970236b3 DM |
945 | $rados->mon_command({ |
946 | prefix => "osd pool set", | |
947 | pool => $param->{name}, | |
948 | var => 'size', | |
949 | val => $size, | |
6e3c2f47 | 950 | format => 'plain', |
970236b3 | 951 | }); |
c7881bf6 | 952 | |
946662f6 | 953 | if (defined($param->{crush_ruleset})) { |
970236b3 DM |
954 | $rados->mon_command({ |
955 | prefix => "osd pool set", | |
956 | pool => $param->{name}, | |
957 | var => 'crush_ruleset', | |
958 | val => $param->{crush_ruleset}, | |
6e3c2f47 | 959 | format => 'plain', |
970236b3 | 960 | }); |
946662f6 DM |
961 | } |
962 | ||
b0537f7b DM |
963 | return undef; |
964 | }}); | |
965 | ||
966 | __PACKAGE__->register_method ({ | |
967 | name => 'destroypool', | |
968 | path => 'pools/{name}', | |
969 | method => 'DELETE', | |
970 | description => "Destroy pool", | |
971 | proxyto => 'node', | |
972 | protected => 1, | |
973 | parameters => { | |
974 | additionalProperties => 0, | |
975 | properties => { | |
976 | node => get_standard_option('pve-node'), | |
977 | name => { | |
978 | description => "The name of the pool. It must be unique.", | |
979 | type => 'string', | |
980 | }, | |
981 | }, | |
982 | }, | |
983 | returns => { type => 'null' }, | |
984 | code => sub { | |
985 | my ($param) = @_; | |
986 | ||
a34866f0 | 987 | PVE::CephTools::check_ceph_inited(); |
b0537f7b | 988 | |
36fd0190 | 989 | my $rados = PVE::RADOS->new(); |
970236b3 DM |
990 | # fixme: '--yes-i-really-really-mean-it' |
991 | $rados->mon_command({ | |
992 | prefix => "osd pool delete", | |
993 | pool => $param->{name}, | |
994 | pool2 => $param->{name}, | |
6e3c2f47 DM |
995 | sure => '--yes-i-really-really-mean-it', |
996 | format => 'plain', | |
997 | }); | |
b0537f7b DM |
998 | |
999 | return undef; | |
1000 | }}); | |
1001 | ||
dd7e2a94 DM |
1002 | __PACKAGE__->register_method ({ |
1003 | name => 'listosd', | |
1004 | path => 'osd', | |
1005 | method => 'GET', | |
1006 | description => "Get Ceph osd list/tree.", | |
1007 | proxyto => 'node', | |
1008 | protected => 1, | |
1009 | parameters => { | |
1010 | additionalProperties => 0, | |
1011 | properties => { | |
1012 | node => get_standard_option('pve-node'), | |
1013 | }, | |
1014 | }, | |
1015 | returns => { | |
1016 | type => "object", | |
1017 | }, | |
1018 | code => sub { | |
1019 | my ($param) = @_; | |
1020 | ||
a34866f0 | 1021 | PVE::CephTools::check_ceph_inited(); |
dd7e2a94 | 1022 | |
36fd0190 | 1023 | my $rados = PVE::RADOS->new(); |
970236b3 | 1024 | my $res = $rados->mon_command({ prefix => 'osd tree' }); |
dd7e2a94 DM |
1025 | |
1026 | die "no tree nodes found\n" if !($res && $res->{nodes}); | |
1027 | ||
1028 | my $nodes = {}; | |
1029 | my $newnodes = {}; | |
1030 | foreach my $e (@{$res->{nodes}}) { | |
1031 | $nodes->{$e->{id}} = $e; | |
1032 | ||
1033 | my $new = { | |
1034 | id => $e->{id}, | |
1035 | name => $e->{name}, | |
1036 | type => $e->{type} | |
1037 | }; | |
1038 | ||
1039 | foreach my $opt (qw(status crush_weight reweight)) { | |
1040 | $new->{$opt} = $e->{$opt} if defined($e->{$opt}); | |
1041 | } | |
1042 | ||
1043 | $newnodes->{$e->{id}} = $new; | |
1044 | } | |
1045 | ||
1046 | foreach my $e (@{$res->{nodes}}) { | |
1047 | my $new = $newnodes->{$e->{id}}; | |
1048 | if ($e->{children} && scalar(@{$e->{children}})) { | |
1049 | $new->{children} = []; | |
1050 | $new->{leaf} = 0; | |
1051 | foreach my $cid (@{$e->{children}}) { | |
1052 | $nodes->{$cid}->{parent} = $e->{id}; | |
1053 | if ($nodes->{$cid}->{type} eq 'osd' && | |
1054 | $e->{type} eq 'host') { | |
1055 | $newnodes->{$cid}->{host} = $e->{name}; | |
1056 | } | |
1057 | push @{$new->{children}}, $newnodes->{$cid}; | |
1058 | } | |
1059 | } else { | |
1060 | $new->{leaf} = ($e->{id} >= 0) ? 1 : 0; | |
1061 | } | |
1062 | } | |
1063 | ||
1064 | my $rootnode; | |
1065 | foreach my $e (@{$res->{nodes}}) { | |
1066 | if (!$nodes->{$e->{id}}->{parent}) { | |
1067 | $rootnode = $newnodes->{$e->{id}}; | |
1068 | last; | |
1069 | } | |
1070 | } | |
1071 | ||
1072 | die "no root node\n" if !$rootnode; | |
1073 | ||
1074 | my $data = { root => $rootnode }; | |
1075 | ||
1076 | return $data; | |
1077 | }}); | |
1078 | ||
38db610a DM |
1079 | __PACKAGE__->register_method ({ |
1080 | name => 'createosd', | |
39e1ad70 | 1081 | path => 'osd', |
38db610a DM |
1082 | method => 'POST', |
1083 | description => "Create OSD", | |
1084 | proxyto => 'node', | |
1085 | protected => 1, | |
1086 | parameters => { | |
1087 | additionalProperties => 0, | |
1088 | properties => { | |
1089 | node => get_standard_option('pve-node'), | |
1090 | dev => { | |
1091 | description => "Block device name.", | |
1092 | type => 'string', | |
43d85563 | 1093 | }, |
0e5816e4 DM |
1094 | journal_dev => { |
1095 | description => "Block device name for journal.", | |
1096 | optional => 1, | |
1097 | type => 'string', | |
1098 | }, | |
43d85563 DM |
1099 | fstype => { |
1100 | description => "File system type.", | |
1101 | type => 'string', | |
1102 | enum => ['xfs', 'ext4', 'btrfs'], | |
1103 | default => 'xfs', | |
1104 | optional => 1, | |
1105 | }, | |
38db610a DM |
1106 | }, |
1107 | }, | |
52d7be41 | 1108 | returns => { type => 'string' }, |
38db610a DM |
1109 | code => sub { |
1110 | my ($param) = @_; | |
1111 | ||
52d7be41 DM |
1112 | my $rpcenv = PVE::RPCEnvironment::get(); |
1113 | ||
1114 | my $authuser = $rpcenv->get_user(); | |
1115 | ||
a34866f0 | 1116 | PVE::CephTools::check_ceph_inited(); |
38db610a | 1117 | |
a34866f0 | 1118 | PVE::CephTools::setup_pve_symlinks(); |
38db610a | 1119 | |
0e5816e4 DM |
1120 | my $journal_dev; |
1121 | ||
1122 | if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) { | |
a34866f0 | 1123 | $journal_dev = PVE::CephTools::verify_blockdev_path($param->{journal_dev}); |
0e5816e4 DM |
1124 | } |
1125 | ||
a34866f0 | 1126 | $param->{dev} = PVE::CephTools::verify_blockdev_path($param->{dev}); |
38db610a | 1127 | |
13f4d762 DM |
1128 | my $disklist = list_disks(); |
1129 | ||
1130 | my $devname = $param->{dev}; | |
1131 | $devname =~ s|/dev/||; | |
1132 | ||
1133 | my $diskinfo = $disklist->{$devname}; | |
1134 | die "unable to get device info for '$devname'\n" | |
1135 | if !$diskinfo; | |
1136 | ||
1137 | die "device '$param->{dev}' is in use\n" | |
1138 | if $diskinfo->{used}; | |
1139 | ||
36fd0190 | 1140 | my $rados = PVE::RADOS->new(); |
970236b3 | 1141 | my $monstat = $rados->mon_command({ prefix => 'mon_status' }); |
38db610a | 1142 | die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid}; |
a34866f0 | 1143 | |
38db610a | 1144 | my $fsid = $monstat->{monmap}->{fsid}; |
2aafa9ea | 1145 | $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/; |
38db610a | 1146 | |
a34866f0 DM |
1147 | my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring'); |
1148 | ||
38db610a | 1149 | if (! -f $ceph_bootstrap_osd_keyring) { |
970236b3 DM |
1150 | my $bindata = $rados->mon_command({ prefix => 'auth get client.bootstrap-osd', format => 'plain' }); |
1151 | PVE::Tools::file_set_contents($ceph_bootstrap_osd_keyring, $bindata); | |
38db610a | 1152 | }; |
2aafa9ea | 1153 | |
52d7be41 DM |
1154 | my $worker = sub { |
1155 | my $upid = shift; | |
38db610a | 1156 | |
43d85563 DM |
1157 | my $fstype = $param->{fstype} || 'xfs'; |
1158 | ||
1159 | print "create OSD on $param->{dev} ($fstype)\n"; | |
52d7be41 | 1160 | |
a34866f0 DM |
1161 | my $ccname = PVE::CephTools::get_config('ccname'); |
1162 | ||
0e5816e4 DM |
1163 | my $cmd = ['ceph-disk', 'prepare', '--zap-disk', '--fs-type', $fstype, |
1164 | '--cluster', $ccname, '--cluster-uuid', $fsid ]; | |
1165 | ||
1166 | if ($journal_dev) { | |
1167 | print "using device '$journal_dev' for journal\n"; | |
1168 | push @$cmd, '--journal-dev', $param->{dev}, $journal_dev; | |
1169 | } else { | |
1170 | push @$cmd, $param->{dev}; | |
1171 | } | |
1172 | ||
1173 | run_command($cmd); | |
52d7be41 DM |
1174 | }; |
1175 | ||
ace49d53 | 1176 | return $rpcenv->fork_worker('cephcreateosd', $devname, $authuser, $worker); |
38db610a DM |
1177 | }}); |
1178 | ||
1179 | __PACKAGE__->register_method ({ | |
1180 | name => 'destroyosd', | |
39e1ad70 DM |
1181 | path => 'osd/{osdid}', |
1182 | method => 'DELETE', | |
38db610a DM |
1183 | description => "Destroy OSD", |
1184 | proxyto => 'node', | |
1185 | protected => 1, | |
1186 | parameters => { | |
1187 | additionalProperties => 0, | |
1188 | properties => { | |
1189 | node => get_standard_option('pve-node'), | |
1190 | osdid => { | |
1191 | description => 'OSD ID', | |
1192 | type => 'integer', | |
1193 | }, | |
0e5816e4 DM |
1194 | cleanup => { |
1195 | description => "If set, we remove partition table entries.", | |
1196 | type => 'boolean', | |
1197 | optional => 1, | |
1198 | default => 0, | |
1199 | }, | |
38db610a DM |
1200 | }, |
1201 | }, | |
52d7be41 | 1202 | returns => { type => 'string' }, |
38db610a DM |
1203 | code => sub { |
1204 | my ($param) = @_; | |
1205 | ||
52d7be41 DM |
1206 | my $rpcenv = PVE::RPCEnvironment::get(); |
1207 | ||
1208 | my $authuser = $rpcenv->get_user(); | |
1209 | ||
a34866f0 | 1210 | PVE::CephTools::check_ceph_inited(); |
38db610a DM |
1211 | |
1212 | my $osdid = $param->{osdid}; | |
1213 | ||
52d7be41 | 1214 | # fixme: not 100% sure what we should do here |
38db610a | 1215 | |
36fd0190 | 1216 | my $rados = PVE::RADOS->new(); |
970236b3 | 1217 | my $stat = $rados->mon_command({ prefix => 'osd dump' }); |
38db610a DM |
1218 | |
1219 | my $osdlist = $stat->{osds} || []; | |
1220 | ||
1221 | my $osdstat; | |
1222 | foreach my $d (@$osdlist) { | |
1223 | if ($d->{osd} == $osdid) { | |
1224 | $osdstat = $d; | |
1225 | last; | |
1226 | } | |
1227 | } | |
1228 | die "no such OSD '$osdid'\n" if !$osdstat; | |
1229 | ||
1230 | die "osd is in use (in == 1)\n" if $osdstat->{in}; | |
1231 | #&$run_ceph_cmd(['osd', 'out', $osdid]); | |
1232 | ||
1233 | die "osd is still runnung (up == 1)\n" if $osdstat->{up}; | |
1234 | ||
cee8e2b6 | 1235 | my $osdsection = "osd.$osdid"; |
38db610a | 1236 | |
52d7be41 DM |
1237 | my $worker = sub { |
1238 | my $upid = shift; | |
38db610a | 1239 | |
2f804640 DM |
1240 | # reopen with longer timeout |
1241 | $rados = PVE::RADOS->new(timeout => $long_rados_timeout); | |
f26b46db | 1242 | |
cee8e2b6 | 1243 | print "destroy OSD $osdsection\n"; |
38db610a | 1244 | |
a34866f0 | 1245 | eval { PVE::CephTools::ceph_service_cmd('stop', $osdsection); }; |
52d7be41 | 1246 | warn $@ if $@; |
38db610a | 1247 | |
52d7be41 | 1248 | print "Remove $osdsection from the CRUSH map\n"; |
6e3c2f47 | 1249 | $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' }); |
38db610a | 1250 | |
52d7be41 | 1251 | print "Remove the $osdsection authentication key.\n"; |
6e3c2f47 | 1252 | $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' }); |
52d7be41 DM |
1253 | |
1254 | print "Remove OSD $osdsection\n"; | |
6e3c2f47 | 1255 | $rados->mon_command({ prefix => "osd rm", ids => "$osdid", format => 'plain' }); |
190df9a1 | 1256 | |
0e5816e4 | 1257 | # try to unmount from standard mount point |
190df9a1 | 1258 | my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid"; |
0e5816e4 DM |
1259 | |
1260 | my $remove_partition = sub { | |
1261 | my ($disklist, $part) = @_; | |
1262 | ||
1263 | return if !$part || (! -b $part ); | |
1264 | ||
1265 | foreach my $real_dev (keys %$disklist) { | |
1266 | my $diskinfo = $disklist->{$real_dev}; | |
1267 | next if !$diskinfo->{gpt}; | |
1268 | if ($part =~ m|^/dev/${real_dev}(\d+)$|) { | |
1269 | my $partnum = $1; | |
1270 | print "remove partition $part (disk '/dev/${real_dev}', partnum $partnum)\n"; | |
1271 | eval { run_command(['/sbin/sgdisk', '-d', $partnum, "/dev/${real_dev}"]); }; | |
1272 | warn $@ if $@; | |
1273 | last; | |
1274 | } | |
1275 | } | |
1276 | }; | |
1277 | ||
1278 | my $journal_part; | |
1279 | my $data_part; | |
1280 | ||
1281 | if ($param->{cleanup}) { | |
1282 | my $jpath = "$mountpoint/journal"; | |
1283 | $journal_part = abs_path($jpath); | |
1284 | ||
1285 | if (my $fd = IO::File->new("/proc/mounts", "r")) { | |
1286 | while (defined(my $line = <$fd>)) { | |
1287 | my ($dev, $path, $fstype) = split(/\s+/, $line); | |
1288 | next if !($dev && $path && $fstype); | |
1289 | next if $dev !~ m|^/dev/|; | |
1290 | if ($path eq $mountpoint) { | |
1291 | $data_part = abs_path($dev); | |
1292 | last; | |
1293 | } | |
1294 | } | |
1295 | close($fd); | |
1296 | } | |
1297 | } | |
1298 | ||
190df9a1 DM |
1299 | print "Unmount OSD $osdsection from $mountpoint\n"; |
1300 | eval { run_command(['umount', $mountpoint]); }; | |
0e5816e4 DM |
1301 | if (my $err = $@) { |
1302 | warn $err; | |
1303 | } elsif ($param->{cleanup}) { | |
1304 | my $disklist = list_disks(); | |
1305 | &$remove_partition($disklist, $journal_part); | |
1306 | &$remove_partition($disklist, $data_part); | |
1307 | } | |
52d7be41 DM |
1308 | }; |
1309 | ||
cee8e2b6 | 1310 | return $rpcenv->fork_worker('cephdestroyosd', $osdsection, $authuser, $worker); |
38db610a | 1311 | }}); |
2f692121 | 1312 | |
a34866f0 | 1313 | |
2f692121 DM |
1314 | __PACKAGE__->register_method ({ |
1315 | name => 'crush', | |
1316 | path => 'crush', | |
1317 | method => 'GET', | |
1318 | description => "Get OSD crush map", | |
1319 | proxyto => 'node', | |
1320 | protected => 1, | |
1321 | parameters => { | |
1322 | additionalProperties => 0, | |
1323 | properties => { | |
1324 | node => get_standard_option('pve-node'), | |
1325 | }, | |
1326 | }, | |
1327 | returns => { type => 'string' }, | |
1328 | code => sub { | |
1329 | my ($param) = @_; | |
1330 | ||
a34866f0 | 1331 | PVE::CephTools::check_ceph_inited(); |
2f692121 | 1332 | |
8b336060 DM |
1333 | # this produces JSON (difficult to read for the user) |
1334 | # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1); | |
2f692121 | 1335 | |
8b336060 DM |
1336 | my $txt = ''; |
1337 | ||
1338 | my $mapfile = "/var/tmp/ceph-crush.map.$$"; | |
1339 | my $mapdata = "/var/tmp/ceph-crush.txt.$$"; | |
1340 | ||
36fd0190 | 1341 | my $rados = PVE::RADOS->new(); |
970236b3 | 1342 | |
8b336060 | 1343 | eval { |
970236b3 DM |
1344 | my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' }); |
1345 | PVE::Tools::file_set_contents($mapfile, $bindata); | |
8b336060 DM |
1346 | run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]); |
1347 | $txt = PVE::Tools::file_get_contents($mapdata); | |
1348 | }; | |
1349 | my $err = $@; | |
1350 | ||
1351 | unlink $mapfile; | |
1352 | unlink $mapdata; | |
1353 | ||
1354 | die $err if $err; | |
1355 | ||
2f692121 DM |
1356 | return $txt; |
1357 | }}); | |
1358 | ||
570278fa DM |
1359 | __PACKAGE__->register_method({ |
1360 | name => 'log', | |
1361 | path => 'log', | |
1362 | method => 'GET', | |
1363 | description => "Read ceph log", | |
1364 | proxyto => 'node', | |
1365 | permissions => { | |
1366 | check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]], | |
1367 | }, | |
1368 | protected => 1, | |
1369 | parameters => { | |
1370 | additionalProperties => 0, | |
1371 | properties => { | |
1372 | node => get_standard_option('pve-node'), | |
1373 | start => { | |
1374 | type => 'integer', | |
1375 | minimum => 0, | |
1376 | optional => 1, | |
1377 | }, | |
1378 | limit => { | |
1379 | type => 'integer', | |
1380 | minimum => 0, | |
1381 | optional => 1, | |
1382 | }, | |
1383 | }, | |
1384 | }, | |
1385 | returns => { | |
1386 | type => 'array', | |
1387 | items => { | |
1388 | type => "object", | |
1389 | properties => { | |
1390 | n => { | |
1391 | description=> "Line number", | |
1392 | type=> 'integer', | |
1393 | }, | |
1394 | t => { | |
1395 | description=> "Line text", | |
1396 | type => 'string', | |
1397 | } | |
1398 | } | |
1399 | } | |
1400 | }, | |
1401 | code => sub { | |
1402 | my ($param) = @_; | |
1403 | ||
1404 | my $rpcenv = PVE::RPCEnvironment::get(); | |
1405 | my $user = $rpcenv->get_user(); | |
1406 | my $node = $param->{node}; | |
1407 | ||
1408 | my $logfile = "/var/log/ceph/ceph.log"; | |
1409 | my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit}); | |
1410 | ||
1411 | $rpcenv->set_result_attrib('total', $count); | |
1412 | ||
1413 | return $lines; | |
1414 | }}); | |
1415 | ||
2f692121 | 1416 |