]> git.proxmox.com Git - pve-manager.git/blob - PVE/API2/Ceph/Pools.pm
fce56787c4c94ed873069977676e47eb77ef88ec
[pve-manager.git] / PVE / API2 / Ceph / Pools.pm
1 package PVE::API2::Ceph::Pools;
2
3 use strict;
4 use warnings;
5
6 use PVE::Ceph::Tools;
7 use PVE::Ceph::Services;
8 use PVE::JSONSchema qw(get_standard_option parse_property_string);
9 use PVE::RADOS;
10 use PVE::RESTHandler;
11 use PVE::RPCEnvironment;
12 use PVE::Storage;
13 use PVE::Tools qw(extract_param);
14
15 use PVE::API2::Storage::Config;
16
17 use base qw(PVE::RESTHandler);
18
19 my $get_autoscale_status = sub {
20 my ($rados) = shift;
21
22 $rados = PVE::RADOS->new() if !defined($rados);
23
24 my $autoscale = $rados->mon_command({
25 prefix => 'osd pool autoscale-status'});
26
27 my $data;
28 foreach my $p (@$autoscale) {
29 $data->{$p->{pool_name}} = $p;
30 }
31
32 return $data;
33 };
34
35
36 __PACKAGE__->register_method ({
37 name => 'lspools',
38 path => '',
39 method => 'GET',
40 description => "List all pools.",
41 proxyto => 'node',
42 protected => 1,
43 permissions => {
44 check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
45 },
46 parameters => {
47 additionalProperties => 0,
48 properties => {
49 node => get_standard_option('pve-node'),
50 },
51 },
52 returns => {
53 type => 'array',
54 items => {
55 type => "object",
56 properties => {
57 pool => {
58 type => 'integer',
59 title => 'ID',
60 },
61 pool_name => {
62 type => 'string',
63 title => 'Name',
64 },
65 size => {
66 type => 'integer',
67 title => 'Size',
68 },
69 type => {
70 type => 'string',
71 title => 'Type',
72 enum => ['replicated', 'erasure', 'unknown'],
73 },
74 min_size => {
75 type => 'integer',
76 title => 'Min Size',
77 },
78 pg_num => {
79 type => 'integer',
80 title => 'PG Num',
81 },
82 pg_num_min => {
83 type => 'integer',
84 title => 'min. PG Num',
85 optional => 1,
86 },
87 pg_num_final => {
88 type => 'integer',
89 title => 'Optimal PG Num',
90 optional => 1,
91 },
92 pg_autoscale_mode => {
93 type => 'string',
94 title => 'PG Autoscale Mode',
95 optional => 1,
96 },
97 crush_rule => {
98 type => 'integer',
99 title => 'Crush Rule',
100 },
101 crush_rule_name => {
102 type => 'string',
103 title => 'Crush Rule Name',
104 },
105 percent_used => {
106 type => 'number',
107 title => '%-Used',
108 },
109 bytes_used => {
110 type => 'integer',
111 title => 'Used',
112 },
113 target_size => {
114 type => 'integer',
115 title => 'PG Autoscale Target Size',
116 optional => 1,
117 },
118 target_size_ratio => {
119 type => 'number',
120 title => 'PG Autoscale Target Ratio',
121 optional => 1,
122 },
123 autoscale_status => {
124 type => 'object',
125 title => 'Autoscale Status',
126 optional => 1,
127 },
128 application_metadata => {
129 type => 'object',
130 title => 'Associated Applications',
131 optional => 1,
132 },
133 },
134 },
135 links => [ { rel => 'child', href => "{pool_name}" } ],
136 },
137 code => sub {
138 my ($param) = @_;
139
140 PVE::Ceph::Tools::check_ceph_inited();
141
142 my $rados = PVE::RADOS->new();
143
144 my $stats = {};
145 my $res = $rados->mon_command({ prefix => 'df' });
146
147 foreach my $d (@{$res->{pools}}) {
148 next if !$d->{stats};
149 next if !defined($d->{id});
150 $stats->{$d->{id}} = $d->{stats};
151 }
152
153 $res = $rados->mon_command({ prefix => 'osd dump' });
154 my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
155
156 my $rules = {};
157 for my $rule (@$rulestmp) {
158 $rules->{$rule->{rule_id}} = $rule->{rule_name};
159 }
160
161 my $data = [];
162 my $attr_list = [
163 'pool',
164 'pool_name',
165 'size',
166 'min_size',
167 'pg_num',
168 'crush_rule',
169 'pg_autoscale_mode',
170 'application_metadata',
171 ];
172
173 # pg_autoscaler module is not enabled in Nautilus
174 my $autoscale = eval { $get_autoscale_status->($rados) };
175
176 foreach my $e (@{$res->{pools}}) {
177 my $d = {};
178 foreach my $attr (@$attr_list) {
179 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
180 }
181
182 if ($autoscale) {
183 $d->{autoscale_status} = $autoscale->{$d->{pool_name}};
184 $d->{pg_num_final} = $d->{autoscale_status}->{pg_num_final};
185 # some info is nested under options instead
186 $d->{pg_num_min} = $e->{options}->{pg_num_min};
187 $d->{target_size} = $e->{options}->{target_size_bytes};
188 $d->{target_size_ratio} = $e->{options}->{target_size_ratio};
189 }
190
191 if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
192 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
193 }
194
195 if (my $s = $stats->{$d->{pool}}) {
196 $d->{bytes_used} = $s->{bytes_used};
197 $d->{percent_used} = $s->{percent_used};
198 }
199
200 # Cephs numerical pool types are barely documented. Found the following in the Ceph
201 # codebase: https://github.com/ceph/ceph/blob/ff144995a849407c258bcb763daa3e03cfce5059/src/osd/osd_types.h#L1221-L1233
202 if ($e->{type} == 1) {
203 $d->{type} = 'replicated';
204 } elsif ($e->{type} == 3) {
205 $d->{type} = 'erasure';
206 } else {
207 # we should never get here, but better be safe
208 $d->{type} = 'unknown';
209 }
210 push @$data, $d;
211 }
212
213
214 return $data;
215 }});
216
217
218 my $ceph_pool_common_options = sub {
219 my ($nodefault) = shift;
220 my $options = {
221 name => {
222 title => 'Name',
223 description => "The name of the pool. It must be unique.",
224 type => 'string',
225 },
226 size => {
227 title => 'Size',
228 description => 'Number of replicas per object',
229 type => 'integer',
230 default => 3,
231 optional => 1,
232 minimum => 1,
233 maximum => 7,
234 },
235 min_size => {
236 title => 'Min Size',
237 description => 'Minimum number of replicas per object',
238 type => 'integer',
239 default => 2,
240 optional => 1,
241 minimum => 1,
242 maximum => 7,
243 },
244 pg_num => {
245 title => 'PG Num',
246 description => "Number of placement groups.",
247 type => 'integer',
248 default => 128,
249 optional => 1,
250 minimum => 1,
251 maximum => 32768,
252 },
253 pg_num_min => {
254 title => 'min. PG Num',
255 description => "Minimal number of placement groups.",
256 type => 'integer',
257 optional => 1,
258 maximum => 32768,
259 },
260 crush_rule => {
261 title => 'Crush Rule Name',
262 description => "The rule to use for mapping object placement in the cluster.",
263 type => 'string',
264 optional => 1,
265 },
266 application => {
267 title => 'Application',
268 description => "The application of the pool.",
269 default => 'rbd',
270 type => 'string',
271 enum => ['rbd', 'cephfs', 'rgw'],
272 optional => 1,
273 },
274 pg_autoscale_mode => {
275 title => 'PG Autoscale Mode',
276 description => "The automatic PG scaling mode of the pool.",
277 type => 'string',
278 enum => ['on', 'off', 'warn'],
279 default => 'warn',
280 optional => 1,
281 },
282 target_size => {
283 description => "The estimated target size of the pool for the PG autoscaler.",
284 title => 'PG Autoscale Target Size',
285 type => 'string',
286 pattern => '^(\d+(\.\d+)?)([KMGT])?$',
287 optional => 1,
288 },
289 target_size_ratio => {
290 description => "The estimated target ratio of the pool for the PG autoscaler.",
291 title => 'PG Autoscale Target Ratio',
292 type => 'number',
293 optional => 1,
294 },
295 };
296
297 if ($nodefault) {
298 delete $options->{$_}->{default} for keys %$options;
299 }
300 return $options;
301 };
302
303
304 my $add_storage = sub {
305 my ($pool, $storeid, $ec_data_pool) = @_;
306
307 my $storage_params = {
308 type => 'rbd',
309 pool => $pool,
310 storage => $storeid,
311 krbd => 0,
312 content => 'rootdir,images',
313 };
314
315 $storage_params->{'data-pool'} = $ec_data_pool if $ec_data_pool;
316
317 PVE::API2::Storage::Config->create($storage_params);
318 };
319
320 my $get_storages = sub {
321 my ($pool) = @_;
322
323 my $cfg = PVE::Storage::config();
324
325 my $storages = $cfg->{ids};
326 my $res = {};
327 foreach my $storeid (keys %$storages) {
328 my $curr = $storages->{$storeid};
329 next if $curr->{type} ne 'rbd';
330 $curr->{pool} = 'rbd' if !defined $curr->{pool}; # set default
331 if (
332 $pool eq $curr->{pool} ||
333 (defined $curr->{'data-pool'} && $pool eq $curr->{'data-pool'})
334 ) {
335 $res->{$storeid} = $storages->{$storeid};
336 }
337 }
338
339 return $res;
340 };
341
342 my $ec_format = {
343 k => {
344 type => 'integer',
345 description => "Number of data chunks. Will create an erasure coded pool plus a"
346 ." replicated pool for metadata.",
347 minimum => 2,
348 },
349 m => {
350 type => 'integer',
351 description => "Number of coding chunks. Will create an erasure coded pool plus a"
352 ." replicated pool for metadata.",
353 minimum => 1,
354 },
355 'failure-domain' => {
356 type => 'string',
357 description => "CRUSH failure domain. Default is 'host'. Will create an erasure"
358 ." coded pool plus a replicated pool for metadata.",
359 format_description => 'domain',
360 optional => 1,
361 default => 'host',
362 },
363 'device-class' => {
364 type => 'string',
365 description => "CRUSH device class. Will create an erasure coded pool plus a"
366 ." replicated pool for metadata.",
367 format_description => 'class',
368 optional => 1,
369 },
370 profile => {
371 description => "Override the erasure code (EC) profile to use. Will create an"
372 ." erasure coded pool plus a replicated pool for metadata.",
373 type => 'string',
374 format_description => 'profile',
375 optional => 1,
376 },
377 };
378
379 sub ec_parse_and_check {
380 my ($property, $rados) = @_;
381 return if !$property;
382
383 my $ec = parse_property_string($ec_format, $property);
384
385 die "Erasure code profile '$ec->{profile}' does not exist.\n"
386 if $ec->{profile} && !PVE::Ceph::Tools::ecprofile_exists($ec->{profile}, $rados);
387
388 return $ec;
389 }
390
391
392 __PACKAGE__->register_method ({
393 name => 'createpool',
394 path => '',
395 method => 'POST',
396 description => "Create Ceph pool",
397 proxyto => 'node',
398 protected => 1,
399 permissions => {
400 check => ['perm', '/', [ 'Sys.Modify' ]],
401 },
402 parameters => {
403 additionalProperties => 0,
404 properties => {
405 node => get_standard_option('pve-node'),
406 add_storages => {
407 description => "Configure VM and CT storage using the new pool.",
408 type => 'boolean',
409 optional => 1,
410 default => "0; for erasure coded pools: 1",
411 },
412 'erasure-coding' => {
413 description => "Create an erasure coded pool for RBD with an accompaning"
414 ." replicated pool for metadata storage. With EC, the common ceph options 'size',"
415 ." 'min_size' and 'crush_rule' parameters will be applied to the metadata pool.",
416 type => 'string',
417 format => $ec_format,
418 optional => 1,
419 },
420 %{ $ceph_pool_common_options->() },
421 },
422 },
423 returns => { type => 'string' },
424 code => sub {
425 my ($param) = @_;
426
427 PVE::Cluster::check_cfs_quorum();
428 PVE::Ceph::Tools::check_ceph_configured();
429
430 my $pool = my $name = extract_param($param, 'name');
431 my $node = extract_param($param, 'node');
432 my $add_storages = extract_param($param, 'add_storages');
433
434 my $rpcenv = PVE::RPCEnvironment::get();
435 my $user = $rpcenv->get_user();
436 # Ceph uses target_size_bytes
437 if (defined($param->{'target_size'})) {
438 my $target_sizestr = extract_param($param, 'target_size');
439 $param->{target_size_bytes} = PVE::JSONSchema::parse_size($target_sizestr);
440 }
441
442 my $rados = PVE::RADOS->new();
443 my $ec = ec_parse_and_check(extract_param($param, 'erasure-coding'), $rados);
444 $add_storages = 1 if $ec && !defined($add_storages);
445
446 if ($add_storages) {
447 $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
448 die "pool name contains characters which are illegal for storage naming\n"
449 if !PVE::JSONSchema::parse_storage_id($pool);
450 }
451
452 # pool defaults
453 $param->{pg_num} //= 128;
454 $param->{size} //= 3;
455 $param->{min_size} //= 2;
456 $param->{application} //= 'rbd';
457 $param->{pg_autoscale_mode} //= 'warn';
458
459 my $worker = sub {
460 # reopen with longer timeout
461 $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
462
463 if ($ec) {
464 if (!$ec->{profile}) {
465 $ec->{profile} = PVE::Ceph::Tools::get_ecprofile_name($pool, $rados);
466 eval {
467 PVE::Ceph::Tools::create_ecprofile(
468 $ec->@{'profile', 'k', 'm', 'failure-domain', 'device-class'},
469 $rados,
470 );
471 };
472 die "could not create erasure code profile '$ec->{profile}': $@\n" if $@;
473 print "created new erasure code profile '$ec->{profile}'\n";
474 }
475
476 my $ec_data_param = {};
477 # copy all params, should be a flat hash
478 $ec_data_param = { map { $_ => $param->{$_} } keys %$param };
479
480 $ec_data_param->{pool_type} = 'erasure';
481 $ec_data_param->{allow_ec_overwrites} = 'true';
482 $ec_data_param->{erasure_code_profile} = $ec->{profile};
483 delete $ec_data_param->{size};
484 delete $ec_data_param->{min_size};
485 delete $ec_data_param->{crush_rule};
486
487 # metadata pool should be ok with 32 PGs
488 $param->{pg_num} = 32;
489
490 $pool = "${name}-metadata";
491 $ec->{data_pool} = "${name}-data";
492
493 PVE::Ceph::Tools::create_pool($ec->{data_pool}, $ec_data_param, $rados);
494 }
495
496 PVE::Ceph::Tools::create_pool($pool, $param, $rados);
497
498 if ($add_storages) {
499 eval { $add_storage->($pool, "${name}", $ec->{data_pool}) };
500 die "adding PVE storage for ceph pool '$name' failed: $@\n" if $@;
501 }
502 };
503
504 return $rpcenv->fork_worker('cephcreatepool', $pool, $user, $worker);
505 }});
506
507
508 __PACKAGE__->register_method ({
509 name => 'destroypool',
510 path => '{name}',
511 method => 'DELETE',
512 description => "Destroy pool",
513 proxyto => 'node',
514 protected => 1,
515 permissions => {
516 check => ['perm', '/', [ 'Sys.Modify' ]],
517 },
518 parameters => {
519 additionalProperties => 0,
520 properties => {
521 node => get_standard_option('pve-node'),
522 name => {
523 description => "The name of the pool. It must be unique.",
524 type => 'string',
525 },
526 force => {
527 description => "If true, destroys pool even if in use",
528 type => 'boolean',
529 optional => 1,
530 default => 0,
531 },
532 remove_storages => {
533 description => "Remove all pveceph-managed storages configured for this pool",
534 type => 'boolean',
535 optional => 1,
536 default => 0,
537 },
538 remove_ecprofile => {
539 description => "Remove the erasure code profile. Defaults to true, if applicable.",
540 type => 'boolean',
541 optional => 1,
542 default => 1,
543 },
544 },
545 },
546 returns => { type => 'string' },
547 code => sub {
548 my ($param) = @_;
549
550 PVE::Ceph::Tools::check_ceph_inited();
551
552 my $rpcenv = PVE::RPCEnvironment::get();
553 my $user = $rpcenv->get_user();
554 $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
555 if $param->{remove_storages};
556
557 my $pool = $param->{name};
558
559 my $worker = sub {
560 my $storages = $get_storages->($pool);
561
562 # if not forced, destroy ceph pool only when no
563 # vm disks are on it anymore
564 if (!$param->{force}) {
565 my $storagecfg = PVE::Storage::config();
566 foreach my $storeid (keys %$storages) {
567 my $storage = $storages->{$storeid};
568
569 # check if any vm disks are on the pool
570 print "checking storage '$storeid' for RBD images..\n";
571 my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
572 die "ceph pool '$pool' still in use by storage '$storeid'\n"
573 if @{$res->{$storeid}} != 0;
574 }
575 }
576 my $rados = PVE::RADOS->new();
577
578 my $pool_properties = PVE::Ceph::Tools::get_pool_properties($pool, $rados);
579
580 PVE::Ceph::Tools::destroy_pool($pool, $rados);
581
582 if (my $ecprofile = $pool_properties->{erasure_code_profile}) {
583 print "found erasure coded profile '$ecprofile', destroying its CRUSH rule\n";
584 my $crush_rule = $pool_properties->{crush_rule};
585 eval { PVE::Ceph::Tools::destroy_crush_rule($crush_rule, $rados); };
586 warn "removing crush rule '${crush_rule}' failed: $@\n" if $@;
587
588 if ($param->{remove_ecprofile} // 1) {
589 print "destroying erasure coded profile '$ecprofile'\n";
590 eval { PVE::Ceph::Tools::destroy_ecprofile($ecprofile, $rados) };
591 warn "removing EC profile '${ecprofile}' failed: $@\n" if $@;
592 }
593 }
594
595 if ($param->{remove_storages}) {
596 my $err;
597 foreach my $storeid (keys %$storages) {
598 # skip external clusters, not managed by pveceph
599 next if $storages->{$storeid}->{monhost};
600 eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
601 if ($@) {
602 warn "failed to remove storage '$storeid': $@\n";
603 $err = 1;
604 }
605 }
606 die "failed to remove (some) storages - check log and remove manually!\n"
607 if $err;
608 }
609 };
610 return $rpcenv->fork_worker('cephdestroypool', $pool, $user, $worker);
611 }});
612
613
614 __PACKAGE__->register_method ({
615 name => 'setpool',
616 path => '{name}',
617 method => 'PUT',
618 description => "Change POOL settings",
619 proxyto => 'node',
620 protected => 1,
621 permissions => {
622 check => ['perm', '/', [ 'Sys.Modify' ]],
623 },
624 parameters => {
625 additionalProperties => 0,
626 properties => {
627 node => get_standard_option('pve-node'),
628 %{ $ceph_pool_common_options->('nodefault') },
629 },
630 },
631 returns => { type => 'string' },
632 code => sub {
633 my ($param) = @_;
634
635 PVE::Ceph::Tools::check_ceph_configured();
636
637 my $rpcenv = PVE::RPCEnvironment::get();
638 my $authuser = $rpcenv->get_user();
639
640 my $pool = extract_param($param, 'name');
641 my $node = extract_param($param, 'node');
642
643 # Ceph uses target_size_bytes
644 if (defined($param->{'target_size'})) {
645 my $target_sizestr = extract_param($param, 'target_size');
646 $param->{target_size_bytes} = PVE::JSONSchema::parse_size($target_sizestr);
647 }
648
649 my $worker = sub {
650 PVE::Ceph::Tools::set_pool($pool, $param);
651 };
652
653 return $rpcenv->fork_worker('cephsetpool', $pool, $authuser, $worker);
654 }});
655
656
657 __PACKAGE__->register_method ({
658 name => 'getpool',
659 path => '{name}',
660 method => 'GET',
661 description => "List pool settings.",
662 proxyto => 'node',
663 protected => 1,
664 permissions => {
665 check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
666 },
667 parameters => {
668 additionalProperties => 0,
669 properties => {
670 node => get_standard_option('pve-node'),
671 name => {
672 description => "The name of the pool. It must be unique.",
673 type => 'string',
674 },
675 verbose => {
676 type => 'boolean',
677 default => 0,
678 optional => 1,
679 description => "If enabled, will display additional data".
680 "(eg. statistics).",
681 },
682 },
683 },
684 returns => {
685 type => "object",
686 properties => {
687 id => { type => 'integer', title => 'ID' },
688 pgp_num => { type => 'integer', title => 'PGP num' },
689 noscrub => { type => 'boolean', title => 'noscrub' },
690 'nodeep-scrub' => { type => 'boolean', title => 'nodeep-scrub' },
691 nodelete => { type => 'boolean', title => 'nodelete' },
692 nopgchange => { type => 'boolean', title => 'nopgchange' },
693 nosizechange => { type => 'boolean', title => 'nosizechange' },
694 write_fadvise_dontneed => { type => 'boolean', title => 'write_fadvise_dontneed' },
695 hashpspool => { type => 'boolean', title => 'hashpspool' },
696 use_gmt_hitset => { type => 'boolean', title => 'use_gmt_hitset' },
697 fast_read => { type => 'boolean', title => 'Fast Read' },
698 application_list => { type => 'array', title => 'Application', optional => 1 },
699 statistics => { type => 'object', title => 'Statistics', optional => 1 },
700 autoscale_status => { type => 'object', title => 'Autoscale Status', optional => 1 },
701 %{ $ceph_pool_common_options->() },
702 },
703 },
704 code => sub {
705 my ($param) = @_;
706
707 PVE::Ceph::Tools::check_ceph_inited();
708
709 my $verbose = $param->{verbose};
710 my $pool = $param->{name};
711
712 my $rados = PVE::RADOS->new();
713 my $res = $rados->mon_command({
714 prefix => 'osd pool get',
715 pool => "$pool",
716 var => 'all',
717 });
718
719 my $data = {
720 id => $res->{pool_id},
721 name => $pool,
722 size => $res->{size},
723 min_size => $res->{min_size},
724 pg_num => $res->{pg_num},
725 pg_num_min => $res->{pg_num_min},
726 pgp_num => $res->{pgp_num},
727 crush_rule => $res->{crush_rule},
728 pg_autoscale_mode => $res->{pg_autoscale_mode},
729 noscrub => "$res->{noscrub}",
730 'nodeep-scrub' => "$res->{'nodeep-scrub'}",
731 nodelete => "$res->{nodelete}",
732 nopgchange => "$res->{nopgchange}",
733 nosizechange => "$res->{nosizechange}",
734 write_fadvise_dontneed => "$res->{write_fadvise_dontneed}",
735 hashpspool => "$res->{hashpspool}",
736 use_gmt_hitset => "$res->{use_gmt_hitset}",
737 fast_read => "$res->{fast_read}",
738 target_size => $res->{target_size_bytes},
739 target_size_ratio => $res->{target_size_ratio},
740 };
741
742 if ($verbose) {
743 my $stats;
744 my $res = $rados->mon_command({ prefix => 'df' });
745
746 # pg_autoscaler module is not enabled in Nautilus
747 # avoid partial read further down, use new rados instance
748 my $autoscale_status = eval { $get_autoscale_status->() };
749 $data->{autoscale_status} = $autoscale_status->{$pool};
750
751 foreach my $d (@{$res->{pools}}) {
752 next if !$d->{stats};
753 next if !defined($d->{name}) && !$d->{name} ne "$pool";
754 $data->{statistics} = $d->{stats};
755 }
756
757 my $apps = $rados->mon_command({ prefix => "osd pool application get", pool => "$pool", });
758 $data->{application_list} = [ keys %$apps ];
759 }
760
761 return $data;
762 }});
763
764
765 1;