]>
git.proxmox.com Git - pve-manager.git/blob - PVE/API2/Ceph/Pool.pm
1 package PVE
::API2
::Ceph
::Pool
;
7 use PVE
::Ceph
::Services
;
8 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
11 use PVE
::RPCEnvironment
;
13 use PVE
::Tools
qw(extract_param);
15 use PVE
::API2
::Storage
::Config
;
17 use base
qw(PVE::RESTHandler);
19 my $get_autoscale_status = sub {
22 $rados = PVE
::RADOS-
>new() if !defined($rados);
24 my $autoscale = $rados->mon_command({
25 prefix
=> 'osd pool autoscale-status'});
28 foreach my $p (@$autoscale) {
29 $data->{$p->{pool_name
}} = $p;
36 __PACKAGE__-
>register_method ({
40 description
=> "List all pools and their settings (which are settable by the POST/PUT endpoints).",
44 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
47 additionalProperties
=> 0,
49 node
=> get_standard_option
('pve-node'),
72 enum
=> ['replicated', 'erasure', 'unknown'],
84 title
=> 'min. PG Num',
89 title
=> 'Optimal PG Num',
92 pg_autoscale_mode
=> {
94 title
=> 'PG Autoscale Mode',
99 title
=> 'Crush Rule',
103 title
=> 'Crush Rule Name',
115 title
=> 'PG Autoscale Target Size',
118 target_size_ratio
=> {
120 title
=> 'PG Autoscale Target Ratio',
123 autoscale_status
=> {
125 title
=> 'Autoscale Status',
128 application_metadata
=> {
130 title
=> 'Associated Applications',
135 links
=> [ { rel
=> 'child', href
=> "{pool_name}" } ],
140 PVE
::Ceph
::Tools
::check_ceph_inited
();
142 my $rados = PVE
::RADOS-
>new();
145 my $res = $rados->mon_command({ prefix
=> 'df' });
147 foreach my $d (@{$res->{pools
}}) {
148 next if !$d->{stats
};
149 next if !defined($d->{id
});
150 $stats->{$d->{id
}} = $d->{stats
};
153 $res = $rados->mon_command({ prefix
=> 'osd dump' });
154 my $rulestmp = $rados->mon_command({ prefix
=> 'osd crush rule dump'});
157 for my $rule (@$rulestmp) {
158 $rules->{$rule->{rule_id
}} = $rule->{rule_name
};
170 'application_metadata',
173 # pg_autoscaler module is not enabled in Nautilus
174 my $autoscale = eval { $get_autoscale_status->($rados) };
176 foreach my $e (@{$res->{pools
}}) {
178 foreach my $attr (@$attr_list) {
179 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
183 $d->{autoscale_status
} = $autoscale->{$d->{pool_name
}};
184 $d->{pg_num_final
} = $d->{autoscale_status
}->{pg_num_final
};
185 # some info is nested under options instead
186 $d->{pg_num_min
} = $e->{options
}->{pg_num_min
};
187 $d->{target_size
} = $e->{options
}->{target_size_bytes
};
188 $d->{target_size_ratio
} = $e->{options
}->{target_size_ratio
};
191 if (defined($d->{crush_rule
}) && defined($rules->{$d->{crush_rule
}})) {
192 $d->{crush_rule_name
} = $rules->{$d->{crush_rule
}};
195 if (my $s = $stats->{$d->{pool
}}) {
196 $d->{bytes_used
} = $s->{bytes_used
};
197 $d->{percent_used
} = $s->{percent_used
};
200 # Cephs numerical pool types are barely documented. Found the following in the Ceph
201 # codebase: https://github.com/ceph/ceph/blob/ff144995a849407c258bcb763daa3e03cfce5059/src/osd/osd_types.h#L1221-L1233
202 if ($e->{type
} == 1) {
203 $d->{type
} = 'replicated';
204 } elsif ($e->{type
} == 3) {
205 $d->{type
} = 'erasure';
207 # we should never get here, but better be safe
208 $d->{type
} = 'unknown';
218 my $ceph_pool_common_options = sub {
219 my ($nodefault) = shift;
223 description
=> "The name of the pool. It must be unique.",
228 description
=> 'Number of replicas per object',
237 description
=> 'Minimum number of replicas per object',
246 description
=> "Number of placement groups.",
254 title
=> 'min. PG Num',
255 description
=> "Minimal number of placement groups.",
261 title
=> 'Crush Rule Name',
262 description
=> "The rule to use for mapping object placement in the cluster.",
267 title
=> 'Application',
268 description
=> "The application of the pool.",
271 enum
=> ['rbd', 'cephfs', 'rgw'],
274 pg_autoscale_mode
=> {
275 title
=> 'PG Autoscale Mode',
276 description
=> "The automatic PG scaling mode of the pool.",
278 enum
=> ['on', 'off', 'warn'],
283 description
=> "The estimated target size of the pool for the PG autoscaler.",
284 title
=> 'PG Autoscale Target Size',
286 pattern
=> '^(\d+(\.\d+)?)([KMGT])?$',
289 target_size_ratio
=> {
290 description
=> "The estimated target ratio of the pool for the PG autoscaler.",
291 title
=> 'PG Autoscale Target Ratio',
298 delete $options->{$_}->{default} for keys %$options;
304 my $add_storage = sub {
305 my ($pool, $storeid, $ec_data_pool) = @_;
307 my $storage_params = {
312 content
=> 'rootdir,images',
315 $storage_params->{'data-pool'} = $ec_data_pool if $ec_data_pool;
317 PVE
::API2
::Storage
::Config-
>create($storage_params);
320 my $get_storages = sub {
323 my $cfg = PVE
::Storage
::config
();
325 my $storages = $cfg->{ids
};
327 foreach my $storeid (keys %$storages) {
328 my $curr = $storages->{$storeid};
329 next if $curr->{type
} ne 'rbd';
330 $curr->{pool
} = 'rbd' if !defined $curr->{pool
}; # set default
332 $pool eq $curr->{pool
} ||
333 (defined $curr->{'data-pool'} && $pool eq $curr->{'data-pool'})
335 $res->{$storeid} = $storages->{$storeid};
345 description
=> "Number of data chunks. Will create an erasure coded pool plus a"
346 ." replicated pool for metadata.",
351 description
=> "Number of coding chunks. Will create an erasure coded pool plus a"
352 ." replicated pool for metadata.",
355 'failure-domain' => {
357 description
=> "CRUSH failure domain. Default is 'host'. Will create an erasure"
358 ." coded pool plus a replicated pool for metadata.",
359 format_description
=> 'domain',
365 description
=> "CRUSH device class. Will create an erasure coded pool plus a"
366 ." replicated pool for metadata.",
367 format_description
=> 'class',
371 description
=> "Override the erasure code (EC) profile to use. Will create an"
372 ." erasure coded pool plus a replicated pool for metadata.",
374 format_description
=> 'profile',
379 sub ec_parse_and_check
{
380 my ($property, $rados) = @_;
381 return if !$property;
383 my $ec = parse_property_string
($ec_format, $property);
385 die "Erasure code profile '$ec->{profile}' does not exist.\n"
386 if $ec->{profile
} && !PVE
::Ceph
::Tools
::ecprofile_exists
($ec->{profile
}, $rados);
392 __PACKAGE__-
>register_method ({
393 name
=> 'createpool',
396 description
=> "Create Ceph pool",
400 check
=> ['perm', '/', [ 'Sys.Modify' ]],
403 additionalProperties
=> 0,
405 node
=> get_standard_option
('pve-node'),
407 description
=> "Configure VM and CT storage using the new pool.",
410 default => "0; for erasure coded pools: 1",
412 'erasure-coding' => {
413 description
=> "Create an erasure coded pool for RBD with an accompaning"
414 ." replicated pool for metadata storage. With EC, the common ceph options 'size',"
415 ." 'min_size' and 'crush_rule' parameters will be applied to the metadata pool.",
417 format
=> $ec_format,
420 %{ $ceph_pool_common_options->() },
423 returns
=> { type
=> 'string' },
427 PVE
::Cluster
::check_cfs_quorum
();
428 PVE
::Ceph
::Tools
::check_ceph_configured
();
430 my $pool = my $name = extract_param
($param, 'name');
431 my $node = extract_param
($param, 'node');
432 my $add_storages = extract_param
($param, 'add_storages');
434 my $rpcenv = PVE
::RPCEnvironment
::get
();
435 my $user = $rpcenv->get_user();
436 # Ceph uses target_size_bytes
437 if (defined($param->{'target_size'})) {
438 my $target_sizestr = extract_param
($param, 'target_size');
439 $param->{target_size_bytes
} = PVE
::JSONSchema
::parse_size
($target_sizestr);
442 my $rados = PVE
::RADOS-
>new();
443 my $ec = ec_parse_and_check
(extract_param
($param, 'erasure-coding'), $rados);
444 $add_storages = 1 if $ec && !defined($add_storages);
447 $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
448 die "pool name contains characters which are illegal for storage naming\n"
449 if !PVE
::JSONSchema
::parse_storage_id
($pool);
453 $param->{pg_num
} //= 128;
454 $param->{size
} //= 3;
455 $param->{min_size
} //= 2;
456 $param->{application
} //= 'rbd';
457 $param->{pg_autoscale_mode
} //= 'warn';
460 # reopen with longer timeout
461 $rados = PVE
::RADOS-
>new(timeout
=> PVE
::Ceph
::Tools
::get_config
('long_rados_timeout'));
464 if (!$ec->{profile
}) {
465 $ec->{profile
} = PVE
::Ceph
::Tools
::get_ecprofile_name
($pool, $rados);
467 PVE
::Ceph
::Tools
::create_ecprofile
(
468 $ec->@{'profile', 'k', 'm', 'failure-domain', 'device-class'},
472 die "could not create erasure code profile '$ec->{profile}': $@\n" if $@;
473 print "created new erasure code profile '$ec->{profile}'\n";
476 my $ec_data_param = {};
477 # copy all params, should be a flat hash
478 $ec_data_param = { map { $_ => $param->{$_} } keys %$param };
480 $ec_data_param->{pool_type
} = 'erasure';
481 $ec_data_param->{allow_ec_overwrites
} = 'true';
482 $ec_data_param->{erasure_code_profile
} = $ec->{profile
};
483 delete $ec_data_param->{size
};
484 delete $ec_data_param->{min_size
};
485 delete $ec_data_param->{crush_rule
};
487 # metadata pool should be ok with 32 PGs
488 $param->{pg_num
} = 32;
490 $pool = "${name}-metadata";
491 $ec->{data_pool
} = "${name}-data";
493 PVE
::Ceph
::Tools
::create_pool
($ec->{data_pool
}, $ec_data_param, $rados);
496 PVE
::Ceph
::Tools
::create_pool
($pool, $param, $rados);
499 eval { $add_storage->($pool, "${name}", $ec->{data_pool
}) };
500 die "adding PVE storage for ceph pool '$name' failed: $@\n" if $@;
504 return $rpcenv->fork_worker('cephcreatepool', $pool, $user, $worker);
508 __PACKAGE__-
>register_method ({
509 name
=> 'destroypool',
512 description
=> "Destroy pool",
516 check
=> ['perm', '/', [ 'Sys.Modify' ]],
519 additionalProperties
=> 0,
521 node
=> get_standard_option
('pve-node'),
523 description
=> "The name of the pool. It must be unique.",
527 description
=> "If true, destroys pool even if in use",
533 description
=> "Remove all pveceph-managed storages configured for this pool",
538 remove_ecprofile
=> {
539 description
=> "Remove the erasure code profile. Defaults to true, if applicable.",
546 returns
=> { type
=> 'string' },
550 PVE
::Ceph
::Tools
::check_ceph_inited
();
552 my $rpcenv = PVE
::RPCEnvironment
::get
();
553 my $user = $rpcenv->get_user();
554 $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
555 if $param->{remove_storages
};
557 my $pool = $param->{name
};
560 my $storages = $get_storages->($pool);
562 # if not forced, destroy ceph pool only when no
563 # vm disks are on it anymore
564 if (!$param->{force
}) {
565 my $storagecfg = PVE
::Storage
::config
();
566 foreach my $storeid (keys %$storages) {
567 my $storage = $storages->{$storeid};
569 # check if any vm disks are on the pool
570 print "checking storage '$storeid' for RBD images..\n";
571 my $res = PVE
::Storage
::vdisk_list
($storagecfg, $storeid);
572 die "ceph pool '$pool' still in use by storage '$storeid'\n"
573 if @{$res->{$storeid}} != 0;
576 my $rados = PVE
::RADOS-
>new();
578 my $pool_properties = PVE
::Ceph
::Tools
::get_pool_properties
($pool, $rados);
580 PVE
::Ceph
::Tools
::destroy_pool
($pool, $rados);
582 if (my $ecprofile = $pool_properties->{erasure_code_profile
}) {
583 print "found erasure coded profile '$ecprofile', destroying its CRUSH rule\n";
584 my $crush_rule = $pool_properties->{crush_rule
};
585 eval { PVE
::Ceph
::Tools
::destroy_crush_rule
($crush_rule, $rados); };
586 warn "removing crush rule '${crush_rule}' failed: $@\n" if $@;
588 if ($param->{remove_ecprofile
} // 1) {
589 print "destroying erasure coded profile '$ecprofile'\n";
590 eval { PVE
::Ceph
::Tools
::destroy_ecprofile
($ecprofile, $rados) };
591 warn "removing EC profile '${ecprofile}' failed: $@\n" if $@;
595 if ($param->{remove_storages
}) {
597 foreach my $storeid (keys %$storages) {
598 # skip external clusters, not managed by pveceph
599 next if $storages->{$storeid}->{monhost
};
600 eval { PVE
::API2
::Storage
::Config-
>delete({storage
=> $storeid}) };
602 warn "failed to remove storage '$storeid': $@\n";
606 die "failed to remove (some) storages - check log and remove manually!\n"
610 return $rpcenv->fork_worker('cephdestroypool', $pool, $user, $worker);
614 __PACKAGE__-
>register_method ({
618 description
=> "Change POOL settings",
622 check
=> ['perm', '/', [ 'Sys.Modify' ]],
625 additionalProperties
=> 0,
627 node
=> get_standard_option
('pve-node'),
628 %{ $ceph_pool_common_options->('nodefault') },
631 returns
=> { type
=> 'string' },
635 PVE
::Ceph
::Tools
::check_ceph_configured
();
637 my $rpcenv = PVE
::RPCEnvironment
::get
();
638 my $authuser = $rpcenv->get_user();
640 my $pool = extract_param
($param, 'name');
641 my $node = extract_param
($param, 'node');
643 # Ceph uses target_size_bytes
644 if (defined($param->{'target_size'})) {
645 my $target_sizestr = extract_param
($param, 'target_size');
646 $param->{target_size_bytes
} = PVE
::JSONSchema
::parse_size
($target_sizestr);
650 PVE
::Ceph
::Tools
::set_pool
($pool, $param);
653 return $rpcenv->fork_worker('cephsetpool', $pool, $authuser, $worker);
656 __PACKAGE__-
>register_method ({
661 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
663 description
=> "Pool index.",
665 additionalProperties
=> 0,
667 node
=> get_standard_option
('pve-node'),
669 description
=> 'The name of the pool.',
680 links
=> [ { rel
=> 'child', href
=> "{name}" } ],
686 { name
=> 'status' },
693 __PACKAGE__-
>register_method ({
695 path
=> '{name}/status',
697 description
=> "Show the current pool status.",
701 check
=> ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any
=> 1],
704 additionalProperties
=> 0,
706 node
=> get_standard_option
('pve-node'),
708 description
=> "The name of the pool. It must be unique.",
715 description
=> "If enabled, will display additional data".
723 id
=> { type
=> 'integer', title
=> 'ID' },
724 pgp_num
=> { type
=> 'integer', title
=> 'PGP num' },
725 noscrub
=> { type
=> 'boolean', title
=> 'noscrub' },
726 'nodeep-scrub' => { type
=> 'boolean', title
=> 'nodeep-scrub' },
727 nodelete
=> { type
=> 'boolean', title
=> 'nodelete' },
728 nopgchange
=> { type
=> 'boolean', title
=> 'nopgchange' },
729 nosizechange
=> { type
=> 'boolean', title
=> 'nosizechange' },
730 write_fadvise_dontneed
=> { type
=> 'boolean', title
=> 'write_fadvise_dontneed' },
731 hashpspool
=> { type
=> 'boolean', title
=> 'hashpspool' },
732 use_gmt_hitset
=> { type
=> 'boolean', title
=> 'use_gmt_hitset' },
733 fast_read
=> { type
=> 'boolean', title
=> 'Fast Read' },
734 application_list
=> { type
=> 'array', title
=> 'Application', optional
=> 1 },
735 statistics
=> { type
=> 'object', title
=> 'Statistics', optional
=> 1 },
736 autoscale_status
=> { type
=> 'object', title
=> 'Autoscale Status', optional
=> 1 },
737 %{ $ceph_pool_common_options->() },
743 PVE
::Ceph
::Tools
::check_ceph_inited
();
745 my $verbose = $param->{verbose
};
746 my $pool = $param->{name
};
748 my $rados = PVE
::RADOS-
>new();
749 my $res = $rados->mon_command({
750 prefix
=> 'osd pool get',
756 id
=> $res->{pool_id
},
758 size
=> $res->{size
},
759 min_size
=> $res->{min_size
},
760 pg_num
=> $res->{pg_num
},
761 pg_num_min
=> $res->{pg_num_min
},
762 pgp_num
=> $res->{pgp_num
},
763 crush_rule
=> $res->{crush_rule
},
764 pg_autoscale_mode
=> $res->{pg_autoscale_mode
},
765 noscrub
=> "$res->{noscrub}",
766 'nodeep-scrub' => "$res->{'nodeep-scrub'}",
767 nodelete
=> "$res->{nodelete}",
768 nopgchange
=> "$res->{nopgchange}",
769 nosizechange
=> "$res->{nosizechange}",
770 write_fadvise_dontneed
=> "$res->{write_fadvise_dontneed}",
771 hashpspool
=> "$res->{hashpspool}",
772 use_gmt_hitset
=> "$res->{use_gmt_hitset}",
773 fast_read
=> "$res->{fast_read}",
774 target_size
=> $res->{target_size_bytes
},
775 target_size_ratio
=> $res->{target_size_ratio
},
780 my $res = $rados->mon_command({ prefix
=> 'df' });
782 # pg_autoscaler module is not enabled in Nautilus
783 # avoid partial read further down, use new rados instance
784 my $autoscale_status = eval { $get_autoscale_status->() };
785 $data->{autoscale_status
} = $autoscale_status->{$pool};
787 foreach my $d (@{$res->{pools
}}) {
788 next if !$d->{stats
};
789 next if !defined($d->{name
}) && !$d->{name
} ne "$pool";
790 $data->{statistics
} = $d->{stats
};
793 my $apps = $rados->mon_command({ prefix
=> "osd pool application get", pool
=> "$pool", });
794 $data->{application_list
} = [ keys %$apps ];