]>
Commit | Line | Data |
---|---|---|
7d4fc5ef | 1 | package PVE::API2::CephOSD; |
38db610a DM |
2 | |
3 | use strict; | |
4 | use warnings; | |
13f4d762 | 5 | use Cwd qw(abs_path); |
38db610a DM |
6 | |
7 | use PVE::SafeSyslog; | |
13f4d762 | 8 | use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach); |
38db610a DM |
9 | use PVE::Exception qw(raise raise_param_exc); |
10 | use PVE::INotify; | |
11 | use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file); | |
12 | use PVE::AccessControl; | |
13 | use PVE::Storage; | |
14 | use PVE::RESTHandler; | |
15 | use PVE::RPCEnvironment; | |
16 | use PVE::JSONSchema qw(get_standard_option); | |
970236b3 | 17 | use PVE::RADOS; |
a34866f0 | 18 | use PVE::CephTools; |
38db610a DM |
19 | |
20 | use base qw(PVE::RESTHandler); | |
21 | ||
22 | use Data::Dumper; # fixme: remove | |
23 | ||
7d4fc5ef DM |
24 | my $get_osd_status = sub { |
25 | my ($rados, $osdid) = @_; | |
0e5816e4 | 26 | |
7d4fc5ef | 27 | my $stat = $rados->mon_command({ prefix => 'osd dump' }); |
2f804640 | 28 | |
7d4fc5ef | 29 | my $osdlist = $stat->{osds} || []; |
38db610a | 30 | |
7d4fc5ef DM |
31 | my $osdstat; |
32 | foreach my $d (@$osdlist) { | |
33 | $osdstat->{$d->{osd}} = $d if defined($d->{osd}); | |
34 | } | |
35 | if (defined($osdid)) { | |
36 | die "no such OSD '$osdid'\n" if !$osdstat->{$osdid}; | |
37 | return $osdstat->{$osdid}; | |
38 | } | |
38db610a | 39 | |
7d4fc5ef DM |
40 | return $osdstat; |
41 | }; | |
13f4d762 | 42 | |
941c0195 DM |
43 | my $get_osd_usage = sub { |
44 | my ($rados) = @_; | |
45 | ||
46 | my $osdlist = $rados->mon_command({ prefix => 'pg dump', | |
47 | dumpcontents => [ 'osds' ]}) || []; | |
48 | ||
49 | my $osdstat; | |
50 | foreach my $d (@$osdlist) { | |
51 | $osdstat->{$d->{osd}} = $d if defined($d->{osd}); | |
52 | } | |
53 | ||
54 | return $osdstat; | |
55 | }; | |
56 | ||
7d4fc5ef DM |
57 | __PACKAGE__->register_method ({ |
58 | name => 'index', | |
59 | path => '', | |
60 | method => 'GET', | |
61 | description => "Get Ceph osd list/tree.", | |
62 | proxyto => 'node', | |
63 | protected => 1, | |
90c75580 TL |
64 | permissions => { |
65 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
66 | }, | |
7d4fc5ef DM |
67 | parameters => { |
68 | additionalProperties => 0, | |
69 | properties => { | |
70 | node => get_standard_option('pve-node'), | |
71 | }, | |
72 | }, | |
73 | # fixme: return a list instead of extjs tree format ? | |
74 | returns => { | |
75 | type => "object", | |
76 | }, | |
77 | code => sub { | |
78 | my ($param) = @_; | |
13f4d762 | 79 | |
7d4fc5ef | 80 | PVE::CephTools::check_ceph_inited(); |
13f4d762 | 81 | |
7d4fc5ef DM |
82 | my $rados = PVE::RADOS->new(); |
83 | my $res = $rados->mon_command({ prefix => 'osd tree' }); | |
13f4d762 | 84 | |
7d4fc5ef | 85 | die "no tree nodes found\n" if !($res && $res->{nodes}); |
13f4d762 | 86 | |
7d4fc5ef | 87 | my $osdhash = &$get_osd_status($rados); |
13f4d762 | 88 | |
941c0195 DM |
89 | my $usagehash = &$get_osd_usage($rados); |
90 | ||
7d4fc5ef DM |
91 | my $nodes = {}; |
92 | my $newnodes = {}; | |
93 | foreach my $e (@{$res->{nodes}}) { | |
94 | $nodes->{$e->{id}} = $e; | |
95 | ||
96 | my $new = { | |
97 | id => $e->{id}, | |
98 | name => $e->{name}, | |
99 | type => $e->{type} | |
100 | }; | |
13f4d762 | 101 | |
7d4fc5ef DM |
102 | foreach my $opt (qw(status crush_weight reweight)) { |
103 | $new->{$opt} = $e->{$opt} if defined($e->{$opt}); | |
104 | } | |
13f4d762 | 105 | |
7d4fc5ef DM |
106 | if (my $stat = $osdhash->{$e->{id}}) { |
107 | $new->{in} = $stat->{in} if defined($stat->{in}); | |
108 | } | |
13f4d762 | 109 | |
941c0195 DM |
110 | if (my $stat = $usagehash->{$e->{id}}) { |
111 | $new->{total_space} = ($stat->{kb} || 1) * 1024; | |
112 | $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024; | |
113 | $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space}; | |
cc81005a DM |
114 | if (my $d = $stat->{fs_perf_stat}) { |
115 | $new->{commit_latency_ms} = $d->{commit_latency_ms}; | |
116 | $new->{apply_latency_ms} = $d->{apply_latency_ms}; | |
117 | } | |
941c0195 DM |
118 | } |
119 | ||
7d4fc5ef | 120 | $newnodes->{$e->{id}} = $new; |
13f4d762 DM |
121 | } |
122 | ||
7d4fc5ef DM |
123 | foreach my $e (@{$res->{nodes}}) { |
124 | my $new = $newnodes->{$e->{id}}; | |
125 | if ($e->{children} && scalar(@{$e->{children}})) { | |
126 | $new->{children} = []; | |
127 | $new->{leaf} = 0; | |
128 | foreach my $cid (@{$e->{children}}) { | |
129 | $nodes->{$cid}->{parent} = $e->{id}; | |
130 | if ($nodes->{$cid}->{type} eq 'osd' && | |
131 | $e->{type} eq 'host') { | |
132 | $newnodes->{$cid}->{host} = $e->{name}; | |
133 | } | |
134 | push @{$new->{children}}, $newnodes->{$cid}; | |
135 | } | |
136 | } else { | |
137 | $new->{leaf} = ($e->{id} >= 0) ? 1 : 0; | |
138 | } | |
0e5816e4 DM |
139 | } |
140 | ||
7d4fc5ef DM |
141 | my $rootnode; |
142 | foreach my $e (@{$res->{nodes}}) { | |
143 | if (!$nodes->{$e->{id}}->{parent}) { | |
144 | $rootnode = $newnodes->{$e->{id}}; | |
145 | last; | |
146 | } | |
0e5816e4 DM |
147 | } |
148 | ||
7d4fc5ef | 149 | die "no root node\n" if !$rootnode; |
13f4d762 | 150 | |
7d4fc5ef | 151 | my $data = { root => $rootnode }; |
0e5816e4 | 152 | |
7d4fc5ef DM |
153 | return $data; |
154 | }}); | |
13f4d762 | 155 | |
7d4fc5ef DM |
156 | __PACKAGE__->register_method ({ |
157 | name => 'createosd', | |
158 | path => '', | |
159 | method => 'POST', | |
160 | description => "Create OSD", | |
161 | proxyto => 'node', | |
162 | protected => 1, | |
163 | parameters => { | |
164 | additionalProperties => 0, | |
165 | properties => { | |
166 | node => get_standard_option('pve-node'), | |
167 | dev => { | |
168 | description => "Block device name.", | |
169 | type => 'string', | |
170 | }, | |
171 | journal_dev => { | |
172 | description => "Block device name for journal.", | |
173 | optional => 1, | |
174 | type => 'string', | |
175 | }, | |
176 | fstype => { | |
177 | description => "File system type.", | |
178 | type => 'string', | |
179 | enum => ['xfs', 'ext4', 'btrfs'], | |
180 | default => 'xfs', | |
181 | optional => 1, | |
182 | }, | |
183 | }, | |
184 | }, | |
185 | returns => { type => 'string' }, | |
186 | code => sub { | |
187 | my ($param) = @_; | |
13f4d762 | 188 | |
7d4fc5ef | 189 | my $rpcenv = PVE::RPCEnvironment::get(); |
13f4d762 | 190 | |
7d4fc5ef | 191 | my $authuser = $rpcenv->get_user(); |
13f4d762 | 192 | |
7d4fc5ef | 193 | PVE::CephTools::check_ceph_inited(); |
0e5816e4 | 194 | |
7d4fc5ef | 195 | PVE::CephTools::setup_pve_symlinks(); |
a7a7fb00 | 196 | |
7d4fc5ef | 197 | my $journal_dev; |
a7a7fb00 | 198 | |
7d4fc5ef DM |
199 | if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) { |
200 | $journal_dev = PVE::CephTools::verify_blockdev_path($param->{journal_dev}); | |
201 | } | |
13f4d762 | 202 | |
7d4fc5ef | 203 | $param->{dev} = PVE::CephTools::verify_blockdev_path($param->{dev}); |
a7a7fb00 | 204 | |
7d4fc5ef | 205 | my $disklist = PVE::CephTools::list_disks(); |
13f4d762 | 206 | |
7d4fc5ef DM |
207 | my $devname = $param->{dev}; |
208 | $devname =~ s|/dev/||; | |
209 | ||
210 | my $diskinfo = $disklist->{$devname}; | |
211 | die "unable to get device info for '$devname'\n" | |
212 | if !$diskinfo; | |
13f4d762 | 213 | |
7d4fc5ef DM |
214 | die "device '$param->{dev}' is in use\n" |
215 | if $diskinfo->{used}; | |
0e5816e4 | 216 | |
7d4fc5ef DM |
217 | my $rados = PVE::RADOS->new(); |
218 | my $monstat = $rados->mon_command({ prefix => 'mon_status' }); | |
219 | die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid}; | |
0e5816e4 | 220 | |
7d4fc5ef DM |
221 | my $fsid = $monstat->{monmap}->{fsid}; |
222 | $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/; | |
0e5816e4 | 223 | |
7d4fc5ef | 224 | my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring'); |
0e5816e4 | 225 | |
7d4fc5ef | 226 | if (! -f $ceph_bootstrap_osd_keyring) { |
1f3e956a | 227 | my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' }); |
7d4fc5ef DM |
228 | PVE::Tools::file_set_contents($ceph_bootstrap_osd_keyring, $bindata); |
229 | }; | |
230 | ||
231 | my $worker = sub { | |
232 | my $upid = shift; | |
0e5816e4 | 233 | |
7d4fc5ef | 234 | my $fstype = $param->{fstype} || 'xfs'; |
0e5816e4 | 235 | |
7d4fc5ef | 236 | print "create OSD on $param->{dev} ($fstype)\n"; |
0e5816e4 | 237 | |
7d4fc5ef | 238 | my $ccname = PVE::CephTools::get_config('ccname'); |
0e5816e4 | 239 | |
7d4fc5ef DM |
240 | my $cmd = ['ceph-disk', 'prepare', '--zap-disk', '--fs-type', $fstype, |
241 | '--cluster', $ccname, '--cluster-uuid', $fsid ]; | |
38db610a | 242 | |
7d4fc5ef DM |
243 | if ($journal_dev) { |
244 | print "using device '$journal_dev' for journal\n"; | |
245 | push @$cmd, '--journal-dev', $param->{dev}, $journal_dev; | |
246 | } else { | |
247 | push @$cmd, $param->{dev}; | |
248 | } | |
249 | ||
250 | run_command($cmd); | |
251 | }; | |
38db610a | 252 | |
7d4fc5ef | 253 | return $rpcenv->fork_worker('cephcreateosd', $devname, $authuser, $worker); |
38db610a DM |
254 | }}); |
255 | ||
13f4d762 | 256 | __PACKAGE__->register_method ({ |
7d4fc5ef DM |
257 | name => 'destroyosd', |
258 | path => '{osdid}', | |
259 | method => 'DELETE', | |
260 | description => "Destroy OSD", | |
13f4d762 DM |
261 | proxyto => 'node', |
262 | protected => 1, | |
263 | parameters => { | |
264 | additionalProperties => 0, | |
265 | properties => { | |
266 | node => get_standard_option('pve-node'), | |
7d4fc5ef DM |
267 | osdid => { |
268 | description => 'OSD ID', | |
269 | type => 'integer', | |
0e5816e4 | 270 | }, |
7d4fc5ef DM |
271 | cleanup => { |
272 | description => "If set, we remove partition table entries.", | |
273 | type => 'boolean', | |
274 | optional => 1, | |
275 | default => 0, | |
13f4d762 DM |
276 | }, |
277 | }, | |
13f4d762 | 278 | }, |
7d4fc5ef | 279 | returns => { type => 'string' }, |
13f4d762 DM |
280 | code => sub { |
281 | my ($param) = @_; | |
282 | ||
7d4fc5ef | 283 | my $rpcenv = PVE::RPCEnvironment::get(); |
13f4d762 | 284 | |
7d4fc5ef | 285 | my $authuser = $rpcenv->get_user(); |
13f4d762 | 286 | |
7d4fc5ef | 287 | PVE::CephTools::check_ceph_inited(); |
0e5816e4 | 288 | |
7d4fc5ef | 289 | my $osdid = $param->{osdid}; |
0e5816e4 | 290 | |
7d4fc5ef DM |
291 | my $rados = PVE::RADOS->new(); |
292 | my $osdstat = &$get_osd_status($rados, $osdid); | |
13f4d762 | 293 | |
7d4fc5ef DM |
294 | die "osd is in use (in == 1)\n" if $osdstat->{in}; |
295 | #&$run_ceph_cmd(['osd', 'out', $osdid]); | |
68e0c4bd | 296 | |
7d4fc5ef | 297 | die "osd is still runnung (up == 1)\n" if $osdstat->{up}; |
68e0c4bd | 298 | |
7d4fc5ef | 299 | my $osdsection = "osd.$osdid"; |
68e0c4bd | 300 | |
7d4fc5ef DM |
301 | my $worker = sub { |
302 | my $upid = shift; | |
68e0c4bd | 303 | |
7d4fc5ef DM |
304 | # reopen with longer timeout |
305 | $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout')); | |
68e0c4bd | 306 | |
7d4fc5ef | 307 | print "destroy OSD $osdsection\n"; |
68e0c4bd | 308 | |
7d4fc5ef DM |
309 | eval { PVE::CephTools::ceph_service_cmd('stop', $osdsection); }; |
310 | warn $@ if $@; | |
68e0c4bd | 311 | |
7d4fc5ef DM |
312 | print "Remove $osdsection from the CRUSH map\n"; |
313 | $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' }); | |
68e0c4bd | 314 | |
7d4fc5ef DM |
315 | print "Remove the $osdsection authentication key.\n"; |
316 | $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' }); | |
317 | ||
318 | print "Remove OSD $osdsection\n"; | |
319 | $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' }); | |
320 | ||
321 | # try to unmount from standard mount point | |
322 | my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid"; | |
323 | ||
324 | my $remove_partition = sub { | |
325 | my ($disklist, $part) = @_; | |
326 | ||
327 | return if !$part || (! -b $part ); | |
328 | ||
329 | foreach my $real_dev (keys %$disklist) { | |
330 | my $diskinfo = $disklist->{$real_dev}; | |
331 | next if !$diskinfo->{gpt}; | |
332 | if ($part =~ m|^/dev/${real_dev}(\d+)$|) { | |
333 | my $partnum = $1; | |
334 | print "remove partition $part (disk '/dev/${real_dev}', partnum $partnum)\n"; | |
335 | eval { run_command(['/sbin/sgdisk', '-d', $partnum, "/dev/${real_dev}"]); }; | |
336 | warn $@ if $@; | |
337 | last; | |
68e0c4bd DM |
338 | } |
339 | } | |
7d4fc5ef | 340 | }; |
68e0c4bd | 341 | |
7d4fc5ef DM |
342 | my $journal_part; |
343 | my $data_part; | |
344 | ||
345 | if ($param->{cleanup}) { | |
346 | my $jpath = "$mountpoint/journal"; | |
347 | $journal_part = abs_path($jpath); | |
348 | ||
349 | if (my $fd = IO::File->new("/proc/mounts", "r")) { | |
350 | while (defined(my $line = <$fd>)) { | |
351 | my ($dev, $path, $fstype) = split(/\s+/, $line); | |
352 | next if !($dev && $path && $fstype); | |
353 | next if $dev !~ m|^/dev/|; | |
354 | if ($path eq $mountpoint) { | |
355 | $data_part = abs_path($dev); | |
356 | last; | |
357 | } | |
358 | } | |
359 | close($fd); | |
68e0c4bd DM |
360 | } |
361 | } | |
7d4fc5ef DM |
362 | |
363 | print "Unmount OSD $osdsection from $mountpoint\n"; | |
364 | eval { run_command(['umount', $mountpoint]); }; | |
365 | if (my $err = $@) { | |
366 | warn $err; | |
367 | } elsif ($param->{cleanup}) { | |
368 | my $disklist = PVE::CephTools::list_disks(); | |
369 | &$remove_partition($disklist, $journal_part); | |
370 | &$remove_partition($disklist, $data_part); | |
371 | } | |
68e0c4bd | 372 | }; |
68e0c4bd | 373 | |
7d4fc5ef | 374 | return $rpcenv->fork_worker('cephdestroyosd', $osdsection, $authuser, $worker); |
68e0c4bd DM |
375 | }}); |
376 | ||
38db610a | 377 | __PACKAGE__->register_method ({ |
7d4fc5ef DM |
378 | name => 'in', |
379 | path => '{osdid}/in', | |
38db610a | 380 | method => 'POST', |
7d4fc5ef | 381 | description => "ceph osd in", |
38db610a DM |
382 | proxyto => 'node', |
383 | protected => 1, | |
90c75580 TL |
384 | permissions => { |
385 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
386 | }, | |
38db610a DM |
387 | parameters => { |
388 | additionalProperties => 0, | |
389 | properties => { | |
390 | node => get_standard_option('pve-node'), | |
7d4fc5ef DM |
391 | osdid => { |
392 | description => 'OSD ID', | |
38db610a | 393 | type => 'integer', |
38db610a DM |
394 | }, |
395 | }, | |
396 | }, | |
7d4fc5ef | 397 | returns => { type => "null" }, |
38db610a DM |
398 | code => sub { |
399 | my ($param) = @_; | |
400 | ||
7d4fc5ef | 401 | PVE::CephTools::check_ceph_inited(); |
38db610a | 402 | |
7d4fc5ef | 403 | my $osdid = $param->{osdid}; |
0e5816e4 | 404 | |
7d4fc5ef | 405 | my $rados = PVE::RADOS->new(); |
f7e342ea | 406 | |
7d4fc5ef | 407 | my $osdstat = &$get_osd_status($rados, $osdid); # osd exists? |
f7e342ea | 408 | |
7d4fc5ef | 409 | my $osdsection = "osd.$osdid"; |
38db610a | 410 | |
7d4fc5ef | 411 | $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' }); |
38db610a DM |
412 | |
413 | return undef; | |
414 | }}); | |
415 | ||
416 | __PACKAGE__->register_method ({ | |
7d4fc5ef DM |
417 | name => 'out', |
418 | path => '{osdid}/out', | |
38db610a | 419 | method => 'POST', |
7d4fc5ef | 420 | description => "ceph osd out", |
38db610a DM |
421 | proxyto => 'node', |
422 | protected => 1, | |
90c75580 TL |
423 | permissions => { |
424 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
425 | }, | |
38db610a DM |
426 | parameters => { |
427 | additionalProperties => 0, | |
428 | properties => { | |
429 | node => get_standard_option('pve-node'), | |
7d4fc5ef DM |
430 | osdid => { |
431 | description => 'OSD ID', | |
432 | type => 'integer', | |
433 | }, | |
38db610a DM |
434 | }, |
435 | }, | |
7d4fc5ef | 436 | returns => { type => "null" }, |
38db610a DM |
437 | code => sub { |
438 | my ($param) = @_; | |
439 | ||
a34866f0 | 440 | PVE::CephTools::check_ceph_inited(); |
38db610a | 441 | |
7d4fc5ef | 442 | my $osdid = $param->{osdid}; |
38db610a | 443 | |
7d4fc5ef | 444 | my $rados = PVE::RADOS->new(); |
38db610a | 445 | |
7d4fc5ef | 446 | my $osdstat = &$get_osd_status($rados, $osdid); # osd exists? |
38db610a | 447 | |
7d4fc5ef | 448 | my $osdsection = "osd.$osdid"; |
38db610a | 449 | |
7d4fc5ef | 450 | $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' }); |
38db610a | 451 | |
7d4fc5ef DM |
452 | return undef; |
453 | }}); | |
38db610a | 454 | |
7d4fc5ef DM |
455 | package PVE::API2::Ceph; |
456 | ||
457 | use strict; | |
458 | use warnings; | |
459 | use File::Basename; | |
460 | use File::Path; | |
461 | use POSIX qw (LONG_MAX); | |
462 | use Cwd qw(abs_path); | |
463 | use IO::Dir; | |
464 | use UUID; | |
465 | use Net::IP; | |
466 | ||
467 | use PVE::SafeSyslog; | |
468 | use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach); | |
469 | use PVE::Exception qw(raise raise_param_exc); | |
470 | use PVE::INotify; | |
471 | use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file); | |
472 | use PVE::AccessControl; | |
473 | use PVE::Storage; | |
474 | use PVE::RESTHandler; | |
475 | use PVE::RPCEnvironment; | |
476 | use PVE::JSONSchema qw(get_standard_option); | |
477 | use JSON; | |
478 | use PVE::RADOS; | |
479 | use PVE::CephTools; | |
480 | ||
481 | use base qw(PVE::RESTHandler); | |
482 | ||
483 | use Data::Dumper; # fixme: remove | |
484 | ||
485 | my $pve_osd_default_journal_size = 1024*5; | |
486 | ||
487 | __PACKAGE__->register_method ({ | |
488 | subclass => "PVE::API2::CephOSD", | |
489 | path => 'osd', | |
490 | }); | |
491 | ||
492 | __PACKAGE__->register_method ({ | |
493 | name => 'index', | |
494 | path => '', | |
495 | method => 'GET', | |
496 | description => "Directory index.", | |
497 | permissions => { user => 'all' }, | |
90c75580 TL |
498 | permissions => { |
499 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
500 | }, | |
7d4fc5ef DM |
501 | parameters => { |
502 | additionalProperties => 0, | |
503 | properties => { | |
504 | node => get_standard_option('pve-node'), | |
505 | }, | |
506 | }, | |
507 | returns => { | |
508 | type => 'array', | |
509 | items => { | |
510 | type => "object", | |
511 | properties => {}, | |
512 | }, | |
513 | links => [ { rel => 'child', href => "{name}" } ], | |
514 | }, | |
515 | code => sub { | |
516 | my ($param) = @_; | |
517 | ||
518 | my $result = [ | |
519 | { name => 'init' }, | |
520 | { name => 'mon' }, | |
521 | { name => 'osd' }, | |
522 | { name => 'pools' }, | |
523 | { name => 'stop' }, | |
524 | { name => 'start' }, | |
525 | { name => 'status' }, | |
526 | { name => 'crush' }, | |
527 | { name => 'config' }, | |
528 | { name => 'log' }, | |
529 | { name => 'disks' }, | |
530 | ]; | |
531 | ||
532 | return $result; | |
533 | }}); | |
534 | ||
535 | __PACKAGE__->register_method ({ | |
536 | name => 'disks', | |
537 | path => 'disks', | |
538 | method => 'GET', | |
539 | description => "List local disks.", | |
540 | proxyto => 'node', | |
541 | protected => 1, | |
90c75580 TL |
542 | permissions => { |
543 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
544 | }, | |
7d4fc5ef DM |
545 | parameters => { |
546 | additionalProperties => 0, | |
547 | properties => { | |
548 | node => get_standard_option('pve-node'), | |
549 | type => { | |
550 | description => "Only list specific types of disks.", | |
551 | type => 'string', | |
552 | enum => ['unused', 'journal_disks'], | |
553 | optional => 1, | |
554 | }, | |
555 | }, | |
556 | }, | |
557 | returns => { | |
558 | type => 'array', | |
559 | items => { | |
560 | type => "object", | |
561 | properties => { | |
562 | dev => { type => 'string' }, | |
563 | used => { type => 'string', optional => 1 }, | |
564 | gpt => { type => 'boolean' }, | |
565 | size => { type => 'integer' }, | |
566 | osdid => { type => 'integer' }, | |
567 | vendor => { type => 'string', optional => 1 }, | |
568 | model => { type => 'string', optional => 1 }, | |
569 | serial => { type => 'string', optional => 1 }, | |
570 | }, | |
571 | }, | |
572 | # links => [ { rel => 'child', href => "{}" } ], | |
573 | }, | |
574 | code => sub { | |
575 | my ($param) = @_; | |
576 | ||
577 | PVE::CephTools::check_ceph_inited(); | |
578 | ||
579 | my $disks = PVE::CephTools::list_disks(); | |
580 | ||
581 | my $res = []; | |
582 | foreach my $dev (keys %$disks) { | |
583 | my $d = $disks->{$dev}; | |
584 | if ($param->{type}) { | |
585 | if ($param->{type} eq 'journal_disks') { | |
586 | next if $d->{osdid} >= 0; | |
587 | next if !$d->{gpt}; | |
588 | } elsif ($param->{type} eq 'unused') { | |
589 | next if $d->{used}; | |
590 | } else { | |
591 | die "internal error"; # should not happen | |
592 | } | |
593 | } | |
594 | ||
595 | $d->{dev} = "/dev/$dev"; | |
596 | push @$res, $d; | |
597 | } | |
598 | ||
599 | return $res; | |
600 | }}); | |
601 | ||
602 | __PACKAGE__->register_method ({ | |
603 | name => 'config', | |
604 | path => 'config', | |
605 | method => 'GET', | |
90c75580 TL |
606 | permissions => { |
607 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
608 | }, | |
7d4fc5ef DM |
609 | description => "Get Ceph configuration.", |
610 | parameters => { | |
611 | additionalProperties => 0, | |
612 | properties => { | |
613 | node => get_standard_option('pve-node'), | |
614 | }, | |
615 | }, | |
616 | returns => { type => 'string' }, | |
617 | code => sub { | |
618 | my ($param) = @_; | |
619 | ||
620 | PVE::CephTools::check_ceph_inited(); | |
621 | ||
622 | my $path = PVE::CephTools::get_config('pve_ceph_cfgpath'); | |
623 | return PVE::Tools::file_get_contents($path); | |
624 | ||
625 | }}); | |
626 | ||
627 | __PACKAGE__->register_method ({ | |
628 | name => 'listmon', | |
629 | path => 'mon', | |
630 | method => 'GET', | |
631 | description => "Get Ceph monitor list.", | |
632 | proxyto => 'node', | |
633 | protected => 1, | |
90c75580 TL |
634 | permissions => { |
635 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
636 | }, | |
7d4fc5ef DM |
637 | parameters => { |
638 | additionalProperties => 0, | |
639 | properties => { | |
640 | node => get_standard_option('pve-node'), | |
641 | }, | |
642 | }, | |
643 | returns => { | |
644 | type => 'array', | |
645 | items => { | |
646 | type => "object", | |
647 | properties => { | |
648 | name => { type => 'string' }, | |
649 | addr => { type => 'string' }, | |
650 | }, | |
651 | }, | |
652 | links => [ { rel => 'child', href => "{name}" } ], | |
653 | }, | |
654 | code => sub { | |
655 | my ($param) = @_; | |
656 | ||
657 | PVE::CephTools::check_ceph_inited(); | |
658 | ||
659 | my $res = []; | |
660 | ||
661 | my $cfg = PVE::CephTools::parse_ceph_config(); | |
662 | ||
663 | my $monhash = {}; | |
664 | foreach my $section (keys %$cfg) { | |
665 | my $d = $cfg->{$section}; | |
666 | if ($section =~ m/^mon\.(\S+)$/) { | |
667 | my $monid = $1; | |
668 | if ($d->{'mon addr'} && $d->{'host'}) { | |
669 | $monhash->{$monid} = { | |
670 | addr => $d->{'mon addr'}, | |
671 | host => $d->{'host'}, | |
672 | name => $monid, | |
673 | } | |
674 | } | |
675 | } | |
676 | } | |
677 | ||
678 | eval { | |
679 | my $rados = PVE::RADOS->new(); | |
680 | my $monstat = $rados->mon_command({ prefix => 'mon_status' }); | |
681 | my $mons = $monstat->{monmap}->{mons}; | |
682 | foreach my $d (@$mons) { | |
683 | next if !defined($d->{name}); | |
684 | $monhash->{$d->{name}}->{rank} = $d->{rank}; | |
685 | $monhash->{$d->{name}}->{addr} = $d->{addr}; | |
686 | if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) { | |
687 | $monhash->{$d->{name}}->{quorum} = 1; | |
688 | } | |
689 | } | |
690 | }; | |
691 | warn $@ if $@; | |
692 | ||
693 | return PVE::RESTHandler::hash_to_array($monhash, 'name'); | |
694 | }}); | |
695 | ||
696 | __PACKAGE__->register_method ({ | |
697 | name => 'init', | |
698 | path => 'init', | |
699 | method => 'POST', | |
700 | description => "Create initial ceph default configuration and setup symlinks.", | |
701 | proxyto => 'node', | |
702 | protected => 1, | |
90c75580 TL |
703 | permissions => { |
704 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
705 | }, | |
7d4fc5ef DM |
706 | parameters => { |
707 | additionalProperties => 0, | |
708 | properties => { | |
709 | node => get_standard_option('pve-node'), | |
710 | network => { | |
711 | description => "Use specific network for all ceph related traffic", | |
712 | type => 'string', format => 'CIDR', | |
713 | optional => 1, | |
714 | maxLength => 128, | |
715 | }, | |
716 | size => { | |
717 | description => 'Number of replicas per object', | |
718 | type => 'integer', | |
719 | default => 2, | |
720 | optional => 1, | |
721 | minimum => 1, | |
722 | maximum => 3, | |
723 | }, | |
724 | pg_bits => { | |
725 | description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)", | |
726 | type => 'integer', | |
727 | default => 6, | |
728 | optional => 1, | |
729 | minimum => 6, | |
730 | maximum => 14, | |
731 | }, | |
732 | }, | |
733 | }, | |
734 | returns => { type => 'null' }, | |
735 | code => sub { | |
736 | my ($param) = @_; | |
737 | ||
738 | PVE::CephTools::check_ceph_installed(); | |
739 | ||
740 | # simply load old config if it already exists | |
741 | my $cfg = PVE::CephTools::parse_ceph_config(); | |
742 | ||
743 | if (!$cfg->{global}) { | |
744 | ||
745 | my $fsid; | |
746 | my $uuid; | |
747 | ||
748 | UUID::generate($uuid); | |
749 | UUID::unparse($uuid, $fsid); | |
750 | ||
751 | $cfg->{global} = { | |
752 | 'fsid' => $fsid, | |
7d4fc5ef DM |
753 | 'auth cluster required' => 'cephx', |
754 | 'auth service required' => 'cephx', | |
755 | 'auth client required' => 'cephx', | |
756 | 'filestore xattr use omap' => 'true', | |
757 | 'osd journal size' => $pve_osd_default_journal_size, | |
758 | 'osd pool default min size' => 1, | |
759 | }; | |
760 | ||
761 | # this does not work for default pools | |
762 | #'osd pool default pg num' => $pg_num, | |
763 | #'osd pool default pgp num' => $pg_num, | |
764 | } | |
765 | ||
766 | $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring'; | |
767 | $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring'; | |
768 | ||
769 | $cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size}; | |
770 | ||
771 | if ($param->{pg_bits}) { | |
772 | $cfg->{global}->{'osd pg bits'} = $param->{pg_bits}; | |
773 | $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits}; | |
774 | } | |
775 | ||
776 | if ($param->{network}) { | |
777 | $cfg->{global}->{'public network'} = $param->{network}; | |
778 | $cfg->{global}->{'cluster network'} = $param->{network}; | |
779 | } | |
780 | ||
781 | PVE::CephTools::write_ceph_config($cfg); | |
782 | ||
783 | PVE::CephTools::setup_pve_symlinks(); | |
784 | ||
785 | return undef; | |
786 | }}); | |
787 | ||
788 | my $find_node_ip = sub { | |
789 | my ($cidr) = @_; | |
790 | ||
7d4fc5ef | 791 | my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n"; |
905c2f51 WB |
792 | my $id = $net->version == 6 ? 'address6' : 'address'; |
793 | ||
794 | my $config = PVE::INotify::read_file('interfaces'); | |
795 | my $ifaces = $config->{ifaces}; | |
7d4fc5ef | 796 | |
905c2f51 | 797 | foreach my $iface (keys %$ifaces) { |
957a59b3 | 798 | my $d = $ifaces->{$iface}; |
905c2f51 WB |
799 | next if !$d->{$id}; |
800 | my $a = Net::IP->new($d->{$id}); | |
7d4fc5ef | 801 | next if !$a; |
905c2f51 | 802 | return $d->{$id} if $net->overlaps($a); |
7d4fc5ef DM |
803 | } |
804 | ||
805 | die "unable to find local address within network '$cidr'\n"; | |
806 | }; | |
807 | ||
808 | __PACKAGE__->register_method ({ | |
809 | name => 'createmon', | |
810 | path => 'mon', | |
811 | method => 'POST', | |
812 | description => "Create Ceph Monitor", | |
813 | proxyto => 'node', | |
814 | protected => 1, | |
90c75580 TL |
815 | permissions => { |
816 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
817 | }, | |
7d4fc5ef DM |
818 | parameters => { |
819 | additionalProperties => 0, | |
820 | properties => { | |
821 | node => get_standard_option('pve-node'), | |
822 | }, | |
823 | }, | |
824 | returns => { type => 'string' }, | |
825 | code => sub { | |
826 | my ($param) = @_; | |
827 | ||
828 | PVE::CephTools::check_ceph_inited(); | |
829 | ||
830 | PVE::CephTools::setup_pve_symlinks(); | |
831 | ||
832 | my $rpcenv = PVE::RPCEnvironment::get(); | |
833 | ||
834 | my $authuser = $rpcenv->get_user(); | |
835 | ||
836 | my $cfg = PVE::CephTools::parse_ceph_config(); | |
837 | ||
838 | my $moncount = 0; | |
839 | ||
840 | my $monaddrhash = {}; | |
841 | ||
842 | foreach my $section (keys %$cfg) { | |
843 | next if $section eq 'global'; | |
844 | my $d = $cfg->{$section}; | |
845 | if ($section =~ m/^mon\./) { | |
846 | $moncount++; | |
847 | if ($d->{'mon addr'}) { | |
848 | $monaddrhash->{$d->{'mon addr'}} = $section; | |
849 | } | |
850 | } | |
851 | } | |
38db610a DM |
852 | |
853 | my $monid; | |
854 | for (my $i = 0; $i < 7; $i++) { | |
855 | if (!$cfg->{"mon.$i"}) { | |
856 | $monid = $i; | |
857 | last; | |
858 | } | |
859 | } | |
860 | die "unable to find usable monitor id\n" if !defined($monid); | |
861 | ||
f7e342ea DM |
862 | my $monsection = "mon.$monid"; |
863 | my $ip; | |
864 | if (my $pubnet = $cfg->{global}->{'public network'}) { | |
865 | $ip = &$find_node_ip($pubnet); | |
866 | } else { | |
867 | $ip = PVE::Cluster::remote_node_ip($param->{node}); | |
868 | } | |
869 | ||
870 | my $monaddr = "$ip:6789"; | |
38db610a DM |
871 | my $monname = $param->{node}; |
872 | ||
873 | die "monitor '$monsection' already exists\n" if $cfg->{$monsection}; | |
874 | die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" | |
875 | if $monaddrhash->{$monaddr}; | |
876 | ||
52d7be41 DM |
877 | my $worker = sub { |
878 | my $upid = shift; | |
38db610a | 879 | |
a34866f0 DM |
880 | my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path'); |
881 | ||
52d7be41 DM |
882 | if (! -f $pve_ckeyring_path) { |
883 | run_command("ceph-authtool $pve_ckeyring_path --create-keyring " . | |
884 | "--gen-key -n client.admin"); | |
885 | } | |
38db610a | 886 | |
a34866f0 | 887 | my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path'); |
52d7be41 DM |
888 | if (! -f $pve_mon_key_path) { |
889 | run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); | |
890 | run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " . | |
891 | "--cap mds 'allow' " . | |
892 | "--cap osd 'allow *' " . | |
893 | "--cap mon 'allow *'"); | |
894 | run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); | |
895 | run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); | |
38db610a DM |
896 | } |
897 | ||
a34866f0 DM |
898 | my $ccname = PVE::CephTools::get_config('ccname'); |
899 | ||
52d7be41 DM |
900 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; |
901 | -d $mondir && die "monitor filesystem '$mondir' already exist\n"; | |
902 | ||
903 | my $monmap = "/tmp/monmap"; | |
904 | ||
905 | eval { | |
906 | mkdir $mondir; | |
907 | ||
908 | if ($moncount > 0) { | |
7d4fc5ef | 909 | my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout')); |
970236b3 DM |
910 | my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' }); |
911 | PVE::Tools::file_set_contents($monmap, $mapdata); | |
52d7be41 DM |
912 | } else { |
913 | run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap"); | |
914 | } | |
38db610a | 915 | |
52d7be41 DM |
916 | run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path"); |
917 | }; | |
918 | my $err = $@; | |
919 | unlink $monmap; | |
920 | if ($err) { | |
921 | File::Path::remove_tree($mondir); | |
922 | die $err; | |
923 | } | |
38db610a | 924 | |
52d7be41 DM |
925 | $cfg->{$monsection} = { |
926 | 'host' => $monname, | |
927 | 'mon addr' => $monaddr, | |
928 | }; | |
38db610a | 929 | |
a34866f0 | 930 | PVE::CephTools::write_ceph_config($cfg); |
38db610a | 931 | |
a34866f0 | 932 | PVE::CephTools::ceph_service_cmd('start', $monsection); |
52d7be41 | 933 | }; |
38db610a | 934 | |
52d7be41 | 935 | return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker); |
38db610a DM |
936 | }}); |
937 | ||
938 | __PACKAGE__->register_method ({ | |
939 | name => 'destroymon', | |
39e1ad70 DM |
940 | path => 'mon/{monid}', |
941 | method => 'DELETE', | |
38db610a DM |
942 | description => "Destroy Ceph monitor.", |
943 | proxyto => 'node', | |
944 | protected => 1, | |
90c75580 TL |
945 | permissions => { |
946 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
947 | }, | |
38db610a DM |
948 | parameters => { |
949 | additionalProperties => 0, | |
950 | properties => { | |
951 | node => get_standard_option('pve-node'), | |
952 | monid => { | |
953 | description => 'Monitor ID', | |
954 | type => 'integer', | |
955 | }, | |
956 | }, | |
957 | }, | |
52d7be41 | 958 | returns => { type => 'string' }, |
38db610a DM |
959 | code => sub { |
960 | my ($param) = @_; | |
961 | ||
52d7be41 DM |
962 | my $rpcenv = PVE::RPCEnvironment::get(); |
963 | ||
964 | my $authuser = $rpcenv->get_user(); | |
965 | ||
a34866f0 | 966 | PVE::CephTools::check_ceph_inited(); |
38db610a | 967 | |
a34866f0 | 968 | my $cfg = PVE::CephTools::parse_ceph_config(); |
38db610a DM |
969 | |
970 | my $monid = $param->{monid}; | |
971 | my $monsection = "mon.$monid"; | |
972 | ||
36fd0190 | 973 | my $rados = PVE::RADOS->new(); |
970236b3 | 974 | my $monstat = $rados->mon_command({ prefix => 'mon_status' }); |
38db610a DM |
975 | my $monlist = $monstat->{monmap}->{mons}; |
976 | ||
977 | die "no such monitor id '$monid'\n" | |
978 | if !defined($cfg->{$monsection}); | |
979 | ||
a34866f0 DM |
980 | my $ccname = PVE::CephTools::get_config('ccname'); |
981 | ||
38db610a DM |
982 | my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; |
983 | -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n"; | |
984 | ||
985 | die "can't remove last monitor\n" if scalar(@$monlist) <= 1; | |
986 | ||
52d7be41 DM |
987 | my $worker = sub { |
988 | my $upid = shift; | |
38db610a | 989 | |
2f804640 | 990 | # reopen with longer timeout |
7d4fc5ef | 991 | $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout')); |
f26b46db | 992 | |
6e3c2f47 | 993 | $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' }); |
38db610a | 994 | |
a34866f0 | 995 | eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); }; |
52d7be41 | 996 | warn $@ if $@; |
38db610a | 997 | |
52d7be41 | 998 | delete $cfg->{$monsection}; |
a34866f0 | 999 | PVE::CephTools::write_ceph_config($cfg); |
52d7be41 DM |
1000 | File::Path::remove_tree($mondir); |
1001 | }; | |
1002 | ||
1003 | return $rpcenv->fork_worker('cephdestroymon', $monsection, $authuser, $worker); | |
38db610a DM |
1004 | }}); |
1005 | ||
1006 | __PACKAGE__->register_method ({ | |
1007 | name => 'stop', | |
1008 | path => 'stop', | |
1009 | method => 'POST', | |
1010 | description => "Stop ceph services.", | |
1011 | proxyto => 'node', | |
1012 | protected => 1, | |
90c75580 TL |
1013 | permissions => { |
1014 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
1015 | }, | |
38db610a DM |
1016 | parameters => { |
1017 | additionalProperties => 0, | |
1018 | properties => { | |
1019 | node => get_standard_option('pve-node'), | |
68e0c4bd DM |
1020 | service => { |
1021 | description => 'Ceph service name.', | |
1022 | type => 'string', | |
1023 | optional => 1, | |
1024 | pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}', | |
1025 | }, | |
38db610a DM |
1026 | }, |
1027 | }, | |
68e0c4bd | 1028 | returns => { type => 'string' }, |
38db610a DM |
1029 | code => sub { |
1030 | my ($param) = @_; | |
1031 | ||
68e0c4bd DM |
1032 | my $rpcenv = PVE::RPCEnvironment::get(); |
1033 | ||
1034 | my $authuser = $rpcenv->get_user(); | |
1035 | ||
a34866f0 | 1036 | PVE::CephTools::check_ceph_inited(); |
38db610a | 1037 | |
a34866f0 | 1038 | my $cfg = PVE::CephTools::parse_ceph_config(); |
38db610a DM |
1039 | scalar(keys %$cfg) || die "no configuration\n"; |
1040 | ||
68e0c4bd DM |
1041 | my $worker = sub { |
1042 | my $upid = shift; | |
38db610a | 1043 | |
68e0c4bd DM |
1044 | my $cmd = ['stop']; |
1045 | if ($param->{service}) { | |
1046 | push @$cmd, $param->{service}; | |
1047 | } | |
1048 | ||
a34866f0 | 1049 | PVE::CephTools::ceph_service_cmd(@$cmd); |
68e0c4bd DM |
1050 | }; |
1051 | ||
1052 | return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph', | |
1053 | $authuser, $worker); | |
38db610a DM |
1054 | }}); |
1055 | ||
1056 | __PACKAGE__->register_method ({ | |
1057 | name => 'start', | |
1058 | path => 'start', | |
1059 | method => 'POST', | |
1060 | description => "Start ceph services.", | |
1061 | proxyto => 'node', | |
1062 | protected => 1, | |
90c75580 TL |
1063 | permissions => { |
1064 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
1065 | }, | |
38db610a DM |
1066 | parameters => { |
1067 | additionalProperties => 0, | |
1068 | properties => { | |
1069 | node => get_standard_option('pve-node'), | |
68e0c4bd DM |
1070 | service => { |
1071 | description => 'Ceph service name.', | |
1072 | type => 'string', | |
1073 | optional => 1, | |
1074 | pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}', | |
1075 | }, | |
38db610a DM |
1076 | }, |
1077 | }, | |
68e0c4bd | 1078 | returns => { type => 'string' }, |
38db610a DM |
1079 | code => sub { |
1080 | my ($param) = @_; | |
1081 | ||
68e0c4bd DM |
1082 | my $rpcenv = PVE::RPCEnvironment::get(); |
1083 | ||
1084 | my $authuser = $rpcenv->get_user(); | |
1085 | ||
a34866f0 | 1086 | PVE::CephTools::check_ceph_inited(); |
38db610a | 1087 | |
a34866f0 | 1088 | my $cfg = PVE::CephTools::parse_ceph_config(); |
38db610a DM |
1089 | scalar(keys %$cfg) || die "no configuration\n"; |
1090 | ||
68e0c4bd DM |
1091 | my $worker = sub { |
1092 | my $upid = shift; | |
38db610a | 1093 | |
68e0c4bd DM |
1094 | my $cmd = ['start']; |
1095 | if ($param->{service}) { | |
1096 | push @$cmd, $param->{service}; | |
1097 | } | |
1098 | ||
a34866f0 | 1099 | PVE::CephTools::ceph_service_cmd(@$cmd); |
68e0c4bd DM |
1100 | }; |
1101 | ||
1102 | return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph', | |
1103 | $authuser, $worker); | |
38db610a DM |
1104 | }}); |
1105 | ||
1106 | __PACKAGE__->register_method ({ | |
1107 | name => 'status', | |
1108 | path => 'status', | |
1109 | method => 'GET', | |
1110 | description => "Get ceph status.", | |
1111 | proxyto => 'node', | |
1112 | protected => 1, | |
90c75580 TL |
1113 | permissions => { |
1114 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
1115 | }, | |
38db610a DM |
1116 | parameters => { |
1117 | additionalProperties => 0, | |
1118 | properties => { | |
1119 | node => get_standard_option('pve-node'), | |
1120 | }, | |
1121 | }, | |
1122 | returns => { type => 'object' }, | |
1123 | code => sub { | |
1124 | my ($param) = @_; | |
1125 | ||
a34866f0 | 1126 | PVE::CephTools::check_ceph_enabled(); |
38db610a | 1127 | |
36fd0190 | 1128 | my $rados = PVE::RADOS->new(); |
970236b3 | 1129 | return $rados->mon_command({ prefix => 'status' }); |
38db610a DM |
1130 | }}); |
1131 | ||
b0537f7b DM |
1132 | __PACKAGE__->register_method ({ |
1133 | name => 'lspools', | |
1134 | path => 'pools', | |
1135 | method => 'GET', | |
1136 | description => "List all pools.", | |
1137 | proxyto => 'node', | |
1138 | protected => 1, | |
90c75580 TL |
1139 | permissions => { |
1140 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
1141 | }, | |
b0537f7b DM |
1142 | parameters => { |
1143 | additionalProperties => 0, | |
1144 | properties => { | |
1145 | node => get_standard_option('pve-node'), | |
1146 | }, | |
1147 | }, | |
1148 | returns => { | |
1149 | type => 'array', | |
1150 | items => { | |
1151 | type => "object", | |
1152 | properties => { | |
1153 | pool => { type => 'integer' }, | |
1154 | pool_name => { type => 'string' }, | |
1155 | size => { type => 'integer' }, | |
1156 | }, | |
1157 | }, | |
1158 | links => [ { rel => 'child', href => "{pool_name}" } ], | |
1159 | }, | |
1160 | code => sub { | |
1161 | my ($param) = @_; | |
1162 | ||
a34866f0 | 1163 | PVE::CephTools::check_ceph_inited(); |
b0537f7b | 1164 | |
36fd0190 | 1165 | my $rados = PVE::RADOS->new(); |
d54f1126 DM |
1166 | |
1167 | my $stats = {}; | |
1168 | my $res = $rados->mon_command({ prefix => 'df' }); | |
6f9ea1c1 WL |
1169 | my $total = $res->{stats}->{total_avail_bytes} || 0; |
1170 | ||
d54f1126 DM |
1171 | foreach my $d (@{$res->{pools}}) { |
1172 | next if !$d->{stats}; | |
1173 | next if !defined($d->{id}); | |
1174 | $stats->{$d->{id}} = $d->{stats}; | |
1175 | } | |
1176 | ||
1177 | $res = $rados->mon_command({ prefix => 'osd dump' }); | |
b0537f7b DM |
1178 | |
1179 | my $data = []; | |
1180 | foreach my $e (@{$res->{pools}}) { | |
1181 | my $d = {}; | |
1182 | foreach my $attr (qw(pool pool_name size min_size pg_num crush_ruleset)) { | |
1183 | $d->{$attr} = $e->{$attr} if defined($e->{$attr}); | |
1184 | } | |
d54f1126 DM |
1185 | if (my $s = $stats->{$d->{pool}}) { |
1186 | $d->{bytes_used} = $s->{bytes_used}; | |
6f9ea1c1 WL |
1187 | $d->{percent_used} = ($s->{bytes_used} / $total)*100 |
1188 | if $s->{max_avail} && $total; | |
d54f1126 | 1189 | } |
b0537f7b DM |
1190 | push @$data, $d; |
1191 | } | |
1192 | ||
d54f1126 | 1193 | |
b0537f7b DM |
1194 | return $data; |
1195 | }}); | |
1196 | ||
1197 | __PACKAGE__->register_method ({ | |
1198 | name => 'createpool', | |
7d4fc5ef | 1199 | path => 'pools', |
38db610a | 1200 | method => 'POST', |
7d4fc5ef | 1201 | description => "Create POOL", |
38db610a DM |
1202 | proxyto => 'node', |
1203 | protected => 1, | |
90c75580 TL |
1204 | permissions => { |
1205 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
1206 | }, | |
38db610a DM |
1207 | parameters => { |
1208 | additionalProperties => 0, | |
1209 | properties => { | |
1210 | node => get_standard_option('pve-node'), | |
7d4fc5ef DM |
1211 | name => { |
1212 | description => "The name of the pool. It must be unique.", | |
38db610a | 1213 | type => 'string', |
43d85563 | 1214 | }, |
7d4fc5ef DM |
1215 | size => { |
1216 | description => 'Number of replicas per object', | |
1217 | type => 'integer', | |
1218 | default => 2, | |
0e5816e4 | 1219 | optional => 1, |
7d4fc5ef DM |
1220 | minimum => 1, |
1221 | maximum => 3, | |
0e5816e4 | 1222 | }, |
7d4fc5ef DM |
1223 | min_size => { |
1224 | description => 'Minimum number of replicas per object', | |
1225 | type => 'integer', | |
1226 | default => 1, | |
1227 | optional => 1, | |
1228 | minimum => 1, | |
1229 | maximum => 3, | |
1230 | }, | |
1231 | pg_num => { | |
1232 | description => "Number of placement groups.", | |
1233 | type => 'integer', | |
1234 | default => 64, | |
1235 | optional => 1, | |
1236 | minimum => 8, | |
1237 | maximum => 32768, | |
1238 | }, | |
1239 | crush_ruleset => { | |
1240 | description => "The ruleset to use for mapping object placement in the cluster.", | |
1241 | type => 'integer', | |
1242 | minimum => 0, | |
1243 | maximum => 32768, | |
1244 | default => 0, | |
43d85563 DM |
1245 | optional => 1, |
1246 | }, | |
38db610a DM |
1247 | }, |
1248 | }, | |
7d4fc5ef | 1249 | returns => { type => 'null' }, |
38db610a DM |
1250 | code => sub { |
1251 | my ($param) = @_; | |
1252 | ||
a34866f0 | 1253 | PVE::CephTools::check_ceph_inited(); |
38db610a | 1254 | |
7d4fc5ef | 1255 | my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path'); |
13f4d762 | 1256 | |
7d4fc5ef DM |
1257 | die "not fully configured - missing '$pve_ckeyring_path'\n" |
1258 | if ! -f $pve_ckeyring_path; | |
13f4d762 | 1259 | |
7d4fc5ef DM |
1260 | my $pg_num = $param->{pg_num} || 64; |
1261 | my $size = $param->{size} || 2; | |
1262 | my $min_size = $param->{min_size} || 1; | |
1263 | my $ruleset = $param->{crush_ruleset} || 0; | |
36fd0190 | 1264 | my $rados = PVE::RADOS->new(); |
38db610a | 1265 | |
7d4fc5ef DM |
1266 | $rados->mon_command({ |
1267 | prefix => "osd pool create", | |
1268 | pool => $param->{name}, | |
1269 | pg_num => int($pg_num), | |
1270 | # this does not work for unknown reason | |
1271 | # properties => ["size=$size", "min_size=$min_size", "crush_ruleset=$ruleset"], | |
1272 | format => 'plain', | |
1273 | }); | |
52d7be41 | 1274 | |
7d4fc5ef DM |
1275 | $rados->mon_command({ |
1276 | prefix => "osd pool set", | |
1277 | pool => $param->{name}, | |
1278 | var => 'min_size', | |
1279 | val => $min_size, | |
1280 | format => 'plain', | |
1281 | }); | |
a34866f0 | 1282 | |
7d4fc5ef DM |
1283 | $rados->mon_command({ |
1284 | prefix => "osd pool set", | |
1285 | pool => $param->{name}, | |
1286 | var => 'size', | |
1287 | val => $size, | |
1288 | format => 'plain', | |
1289 | }); | |
0e5816e4 | 1290 | |
7d4fc5ef DM |
1291 | if (defined($param->{crush_ruleset})) { |
1292 | $rados->mon_command({ | |
1293 | prefix => "osd pool set", | |
1294 | pool => $param->{name}, | |
1295 | var => 'crush_ruleset', | |
1296 | val => $param->{crush_ruleset}, | |
1297 | format => 'plain', | |
1298 | }); | |
1299 | } | |
52d7be41 | 1300 | |
7d4fc5ef | 1301 | return undef; |
38db610a DM |
1302 | }}); |
1303 | ||
1304 | __PACKAGE__->register_method ({ | |
7d4fc5ef DM |
1305 | name => 'destroypool', |
1306 | path => 'pools/{name}', | |
39e1ad70 | 1307 | method => 'DELETE', |
7d4fc5ef | 1308 | description => "Destroy pool", |
38db610a DM |
1309 | proxyto => 'node', |
1310 | protected => 1, | |
90c75580 TL |
1311 | permissions => { |
1312 | check => ['perm', '/', [ 'Sys.Modify' ]], | |
1313 | }, | |
38db610a DM |
1314 | parameters => { |
1315 | additionalProperties => 0, | |
1316 | properties => { | |
1317 | node => get_standard_option('pve-node'), | |
7d4fc5ef DM |
1318 | name => { |
1319 | description => "The name of the pool. It must be unique.", | |
1320 | type => 'string', | |
0e5816e4 | 1321 | }, |
38db610a DM |
1322 | }, |
1323 | }, | |
7d4fc5ef | 1324 | returns => { type => 'null' }, |
38db610a DM |
1325 | code => sub { |
1326 | my ($param) = @_; | |
1327 | ||
a34866f0 | 1328 | PVE::CephTools::check_ceph_inited(); |
38db610a | 1329 | |
36fd0190 | 1330 | my $rados = PVE::RADOS->new(); |
7d4fc5ef DM |
1331 | # fixme: '--yes-i-really-really-mean-it' |
1332 | $rados->mon_command({ | |
1333 | prefix => "osd pool delete", | |
1334 | pool => $param->{name}, | |
1335 | pool2 => $param->{name}, | |
1336 | sure => '--yes-i-really-really-mean-it', | |
1337 | format => 'plain', | |
1338 | }); | |
52d7be41 | 1339 | |
7d4fc5ef | 1340 | return undef; |
38db610a | 1341 | }}); |
2f692121 | 1342 | |
a34866f0 | 1343 | |
2f692121 DM |
1344 | __PACKAGE__->register_method ({ |
1345 | name => 'crush', | |
1346 | path => 'crush', | |
1347 | method => 'GET', | |
1348 | description => "Get OSD crush map", | |
1349 | proxyto => 'node', | |
1350 | protected => 1, | |
90c75580 TL |
1351 | permissions => { |
1352 | check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], | |
1353 | }, | |
2f692121 DM |
1354 | parameters => { |
1355 | additionalProperties => 0, | |
1356 | properties => { | |
1357 | node => get_standard_option('pve-node'), | |
1358 | }, | |
1359 | }, | |
1360 | returns => { type => 'string' }, | |
1361 | code => sub { | |
1362 | my ($param) = @_; | |
1363 | ||
a34866f0 | 1364 | PVE::CephTools::check_ceph_inited(); |
2f692121 | 1365 | |
8b336060 DM |
1366 | # this produces JSON (difficult to read for the user) |
1367 | # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1); | |
2f692121 | 1368 | |
8b336060 DM |
1369 | my $txt = ''; |
1370 | ||
1371 | my $mapfile = "/var/tmp/ceph-crush.map.$$"; | |
1372 | my $mapdata = "/var/tmp/ceph-crush.txt.$$"; | |
1373 | ||
36fd0190 | 1374 | my $rados = PVE::RADOS->new(); |
970236b3 | 1375 | |
8b336060 | 1376 | eval { |
970236b3 DM |
1377 | my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' }); |
1378 | PVE::Tools::file_set_contents($mapfile, $bindata); | |
8b336060 DM |
1379 | run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]); |
1380 | $txt = PVE::Tools::file_get_contents($mapdata); | |
1381 | }; | |
1382 | my $err = $@; | |
1383 | ||
1384 | unlink $mapfile; | |
1385 | unlink $mapdata; | |
1386 | ||
1387 | die $err if $err; | |
1388 | ||
2f692121 DM |
1389 | return $txt; |
1390 | }}); | |
1391 | ||
570278fa DM |
1392 | __PACKAGE__->register_method({ |
1393 | name => 'log', | |
1394 | path => 'log', | |
1395 | method => 'GET', | |
1396 | description => "Read ceph log", | |
1397 | proxyto => 'node', | |
1398 | permissions => { | |
1399 | check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]], | |
1400 | }, | |
1401 | protected => 1, | |
1402 | parameters => { | |
1403 | additionalProperties => 0, | |
1404 | properties => { | |
1405 | node => get_standard_option('pve-node'), | |
1406 | start => { | |
1407 | type => 'integer', | |
1408 | minimum => 0, | |
1409 | optional => 1, | |
1410 | }, | |
1411 | limit => { | |
1412 | type => 'integer', | |
1413 | minimum => 0, | |
1414 | optional => 1, | |
1415 | }, | |
1416 | }, | |
1417 | }, | |
1418 | returns => { | |
1419 | type => 'array', | |
1420 | items => { | |
1421 | type => "object", | |
1422 | properties => { | |
1423 | n => { | |
1424 | description=> "Line number", | |
1425 | type=> 'integer', | |
1426 | }, | |
1427 | t => { | |
1428 | description=> "Line text", | |
1429 | type => 'string', | |
1430 | } | |
1431 | } | |
1432 | } | |
1433 | }, | |
1434 | code => sub { | |
1435 | my ($param) = @_; | |
1436 | ||
1437 | my $rpcenv = PVE::RPCEnvironment::get(); | |
1438 | my $user = $rpcenv->get_user(); | |
1439 | my $node = $param->{node}; | |
1440 | ||
1441 | my $logfile = "/var/log/ceph/ceph.log"; | |
1442 | my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit}); | |
1443 | ||
1444 | $rpcenv->set_result_attrib('total', $count); | |
1445 | ||
1446 | return $lines; | |
1447 | }}); | |
1448 | ||
2f692121 | 1449 |