[pve-manager.git] / PVE / CLI / pveceph.pm

package PVE::CLI::pveceph;

use strict;
use warnings;

use Fcntl ':flock';
use File::Path;
use IO::File;
use JSON;
use Data::Dumper;
use LWP::UserAgent;

use PVE::SafeSyslog;
use PVE::Cluster;
use PVE::INotify;
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::Tools qw(run_command);
use PVE::JSONSchema qw(get_standard_option);
use PVE::Ceph::Tools;
use PVE::Ceph::Services;
use PVE::API2::Ceph;
use PVE::API2::Ceph::FS;
use PVE::API2::Ceph::MDS;
use PVE::API2::Ceph::MGR;
use PVE::API2::Ceph::MON;
use PVE::API2::Ceph::OSD;

use PVE::CLIHandler;

use base qw(PVE::CLIHandler);

my $nodename = PVE::INotify::nodename();

my $upid_exit = sub {
    my $upid = shift;
    my $status = PVE::Tools::upid_read_status($upid);
    exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
};

sub setup_environment {
    PVE::RPCEnvironment->setup_default_cli_env();
}

__PACKAGE__->register_method ({
    name => 'purge',
    path => 'purge',
    method => 'POST',
    description => "Destroy ceph related data and configuration files.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    logs => {
		description => 'Additionally purge Ceph logs, /var/log/ceph.',
		type => 'boolean',
		optional => 1,
	    },
	    crash => {
		description => 'Additionally purge Ceph crash logs, /var/lib/ceph/crash.',
		type => 'boolean',
		optional => 1,
	    },
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	my $message;
	my $pools = [];
	my $monstat = {};
	my $mdsstat = {};
	my $osdstat = [];

	eval {
	    my $rados = PVE::RADOS->new();
	    $pools = PVE::Ceph::Tools::ls_pools(undef, $rados);
	    $monstat = PVE::Ceph::Services::get_services_info('mon', undef, $rados);
	    $mdsstat = PVE::Ceph::Services::get_services_info('mds', undef, $rados);
	    $osdstat = $rados->mon_command({ prefix => 'osd metadata' });
	};
	warn "Error gathering ceph info, already purged? Message: $@" if $@;

	my $osd = grep { $_->{hostname} eq $nodename } @$osdstat;
	my $mds = grep { $mdsstat->{$_}->{host} eq $nodename } keys %$mdsstat;
	my $mon = grep { $monstat->{$_}->{host} eq $nodename } keys %$monstat;

	# no pools = no data
	$message .= "- remove pools, this will !!DESTROY DATA!!\n" if @$pools;
	$message .= "- remove active OSD on $nodename\n" if $osd;
	$message .= "- remove active MDS on $nodename\n" if $mds;
	$message .= "- remove other MONs, $nodename is not the last MON\n"
	    if scalar(keys %$monstat) > 1 && $mon;

	# display all steps at once
	die "Unable to purge Ceph!\n\nTo continue:\n$message" if $message;

	my $services = PVE::Ceph::Services::get_local_services();
	$services->{mon} = $monstat if $mon;
	$services->{crash}->{$nodename} = { direxists => 1 } if $param->{crash};
	$services->{logs}->{$nodename} = { direxists => 1 } if $param->{logs};

	PVE::Ceph::Tools::purge_all_ceph_services($services);
	PVE::Ceph::Tools::purge_all_ceph_files($services);

	return undef;
    }});

my $supported_ceph_versions = ['octopus', 'pacific', 'quincy'];
my $default_ceph_version = 'pacific';

__PACKAGE__->register_method ({
    name => 'install',
    path => 'install',
    method => 'POST',
    description => "Install ceph related packages.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    version => {
		type => 'string',
		enum => $supported_ceph_versions,
		default => $default_ceph_version,
		description => "Ceph version to install.",
		optional => 1,
	    },
	    'allow-experimental' => {
		type => 'boolean',
		default => 0,
		optional => 1,
		description => "Allow experimental versions. Use with care!",
	    },
	    'test-repository' => {
		type => 'boolean',
		default => 0,
		optional => 1,
		description => "Use the test, not the main repository. Use with care!",
	    },
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	my $cephver = $param->{version} || $default_ceph_version;

	my $repo = $param->{'test-repository'} ? 'test' : 'main';

	my $repolist;
	if ($cephver eq 'octopus') {
	    warn "Ceph Octopus will go EOL after 2022-07\n";
	    $repolist = "deb http://download.proxmox.com/debian/ceph-octopus bullseye $repo\n";
	} elsif ($cephver eq 'pacific') {
	    $repolist = "deb http://download.proxmox.com/debian/ceph-pacific bullseye $repo\n";
	} elsif ($cephver eq 'quincy') {
	    $repolist = "deb http://download.proxmox.com/debian/ceph-quincy bullseye $repo\n";
	} else {
	    die "unsupported ceph version: $cephver";
	}
	PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $repolist);

	my $supported_re = join('|', $supported_ceph_versions->@*);
	warn "WARNING: installing non-default ceph release '$cephver'!\n" if $cephver !~ qr/^(?:$supported_re)$/;

	local $ENV{DEBIAN_FRONTEND} = 'noninteractive';
	print "update available package list\n";
	eval {
	    run_command(
		['apt-get', '-q', 'update'],
		outfunc => sub {},
		errfunc => sub { print STDERR "$_[0]\n" },
	    )
	};

	my @apt_install = qw(apt-get --no-install-recommends -o Dpkg::Options::=--force-confnew install --);
	my @ceph_packages = qw(
	    ceph
	    ceph-common
	    ceph-mds
	    ceph-fuse
	    gdisk
	    nvme-cli
	);

	# got split out with quincy and is required by PVE tooling, conditionally exclude it for older
	# FIXME: remove condition with PVE 8.0, i.e., once we only support quincy+ new installations
	if ($cephver ne 'octopus' and $cephver ne 'pacific') {
	    push @ceph_packages, 'ceph-volume';
	}

	print "start installation\n";

	# this flag helps to determine when apt is actually done installing (vs. partial extracing)
	my $install_flag_fn = PVE::Ceph::Tools::ceph_install_flag_file();
	open(my $install_flag, '>', $install_flag_fn) or die "could not create install flag - $!\n";
	close $install_flag;

	if (system(@apt_install, @ceph_packages) != 0) {
	    unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
	    die "apt failed during ceph installation ($?)\n";
	}

	print "\ninstalled ceph $cephver successfully!\n";
	# done: drop flag file so that the PVE::Ceph::Tools check returns Ok now.
	unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";

	print "\nreloading API to load new Ceph RADOS library...\n";
	run_command([
	    'systemctl', 'try-reload-or-restart', 'pvedaemon.service', 'pveproxy.service'
	]);

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'status',
    path => 'status',
    method => 'GET',
    description => "Get Ceph Status.",
    parameters => {
	additionalProperties => 0,
    },
    returns => { type => 'null' },
    code => sub {
	PVE::Ceph::Tools::check_ceph_inited();

	run_command(
	    ['ceph', '-s'],
	    outfunc => sub { print "$_[0]\n" },
	    errfunc => sub { print STDERR "$_[0]\n" },
	    timeout => 15,
	);
	return undef;
    }});

my $get_storages = sub {
    my ($fs, $is_default) = @_;

    my $cfg = PVE::Storage::config();

    my $storages = $cfg->{ids};
    my $res = {};
    foreach my $storeid (keys %$storages) {
	my $curr = $storages->{$storeid};
	next if $curr->{type} ne 'cephfs';
	my $cur_fs = $curr->{'fs-name'};
	$res->{$storeid} = $storages->{$storeid}
	    if (!defined($cur_fs) && $is_default) || (defined($cur_fs) && $fs eq $cur_fs);
    }

    return $res;
};

__PACKAGE__->register_method ({
    name => 'destroyfs',
    path => 'destroyfs',
    method => 'DELETE',
    description => "Destroy a Ceph filesystem",
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    name => {
		description => "The ceph filesystem name.",
		type => 'string',
	    },
	    'remove-storages' => {
		description => "Remove all pveceph-managed storages configured for this fs.",
		type => 'boolean',
		optional => 1,
		default => 0,
	    },
	    'remove-pools' => {
		description => "Remove data and metadata pools configured for this fs.",
		type => 'boolean',
		optional => 1,
		default => 0,
	    },
	},
    },
    returns => { type => 'string' },
    code => sub {
	my ($param) = @_;

	PVE::Ceph::Tools::check_ceph_inited();

	my $rpcenv = PVE::RPCEnvironment::get();
	my $user = $rpcenv->get_user();

	my $fs_name = $param->{name};

	my $fs;
	my $fs_list = PVE::Ceph::Tools::ls_fs();
	for my $entry (@$fs_list) {
	    next if $entry->{name} ne $fs_name;
	    $fs = $entry;
	    last;
	}
	die "no such cephfs '$fs_name'\n" if !$fs;

	my $worker = sub {
	    my $rados = PVE::RADOS->new();

	    if ($param->{'remove-storages'}) {
		my $defaultfs;
		my $fs_dump = $rados->mon_command({ prefix => "fs dump" });
		for my $fs ($fs_dump->{filesystems}->@*) {
		    next if $fs->{id} != $fs_dump->{default_fscid};
		    $defaultfs = $fs->{mdsmap}->{fs_name};
		}
		warn "no default fs found, maybe not all relevant storages are removed\n"
		    if !defined($defaultfs);

		my $storages = $get_storages->($fs_name, $fs_name eq ($defaultfs // ''));
		for my $storeid (keys %$storages) {
		    my $store = $storages->{$storeid};
		    if (!$store->{disable}) {
			die "storage '$storeid' is not disabled, make sure to disable ".
			    "and unmount the storage first\n";
		    }
		}

		my $err;
		for my $storeid (keys %$storages) {
		    # skip external clusters, not managed by pveceph
		    next if $storages->{$storeid}->{monhost};
		    eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
		    if ($@) {
			warn "failed to remove storage '$storeid': $@\n";
			$err = 1;
		    }
		}
		die "failed to remove (some) storages - check log and remove manually!\n"
		    if $err;
	    }

	    PVE::Ceph::Tools::destroy_fs($fs_name, $rados);

	    if ($param->{'remove-pools'}) {
		warn "removing metadata pool '$fs->{metadata_pool}'\n";
		eval { PVE::Ceph::Tools::destroy_pool($fs->{metadata_pool}, $rados) };
		warn "$@\n" if $@;

		foreach my $pool ($fs->{data_pools}->@*) {
		    warn "removing data pool '$pool'\n";
		    eval { PVE::Ceph::Tools::destroy_pool($pool, $rados) };
		    warn "$@\n" if $@;
		}
	    }

	};
	return $rpcenv->fork_worker('cephdestroyfs', $fs_name,  $user, $worker);
    }});

our $cmddef = {
    init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ],
    pool => {
	ls => [ 'PVE::API2::Ceph::Pool', 'lspools', [], { node => $nodename }, sub {
	    my ($data, $schema, $options) = @_;
	    PVE::CLIFormatter::print_api_result($data, $schema,
		[
		    'pool_name',
		    'size',
		    'min_size',
		    'pg_num',
		    'pg_num_min',
		    'pg_num_final',
		    'pg_autoscale_mode',
		    'target_size',
		    'target_size_ratio',
		    'crush_rule_name',
		    'percent_used',
		    'bytes_used',
		],
		$options);
	}, $PVE::RESTHandler::standard_output_options],
	create => [ 'PVE::API2::Ceph::Pool', 'createpool', ['name'], { node => $nodename }],
	destroy => [ 'PVE::API2::Ceph::Pool', 'destroypool', ['name'], { node => $nodename } ],
	set => [ 'PVE::API2::Ceph::Pool', 'setpool', ['name'], { node => $nodename } ],
	get => [ 'PVE::API2::Ceph::Pool', 'getpool', ['name'], { node => $nodename }, sub {
	    my ($data, $schema, $options) = @_;
	    PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
	}, $PVE::RESTHandler::standard_output_options],
    },
    lspools => { alias => 'pool ls' },
    createpool => { alias => 'pool create' },
    destroypool => { alias => 'pool destroy' },
    fs => {
	create => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
	destroy => [ __PACKAGE__, 'destroyfs', ['name'], { node => $nodename }],
    },
    osd => {
	create => [ 'PVE::API2::Ceph::OSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
	destroy => [ 'PVE::API2::Ceph::OSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
    },
    createosd => { alias => 'osd create' },
    destroyosd => { alias => 'osd destroy' },
    mon => {
	create => [ 'PVE::API2::Ceph::MON', 'createmon', [], { node => $nodename }, $upid_exit],
	destroy => [ 'PVE::API2::Ceph::MON', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
    },
    createmon => { alias => 'mon create' },
    destroymon => { alias => 'mon destroy' },
    mgr => {
	create => [ 'PVE::API2::Ceph::MGR', 'createmgr', [], { node => $nodename }, $upid_exit],
	destroy => [ 'PVE::API2::Ceph::MGR', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
    },
    createmgr => { alias => 'mgr create' },
    destroymgr => { alias => 'mgr destroy' },
    mds => {
	create => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
	destroy => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['name'], { node => $nodename }, $upid_exit],
    },
    start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename }, $upid_exit],
    stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename }, $upid_exit],
    install => [ __PACKAGE__, 'install', [] ],
    purge => [  __PACKAGE__, 'purge', [] ],
    status => [ __PACKAGE__, 'status', []],
};

1;
Commit	Line	Data
3e8560ac DM	1	package PVE::CLI::pveceph;
	2
	3	use strict;
	4	use warnings;
	5
	6	use Fcntl ':flock';
	7	use File::Path;
	8	use IO::File;
	9	use JSON;
	10	use Data::Dumper;
	11	use LWP::UserAgent;
	12
	13	use PVE::SafeSyslog;
	14	use PVE::Cluster;
	15	use PVE::INotify;
	16	use PVE::RPCEnvironment;
	17	use PVE::Storage;
	18	use PVE::Tools qw(run_command);
	19	use PVE::JSONSchema qw(get_standard_option);
6fb08cb9	20	use PVE::Ceph::Tools;
91dfa228	21	use PVE::Ceph::Services;
3e8560ac	22	use PVE::API2::Ceph;
7e1a9d25	23	use PVE::API2::Ceph::FS;
b82649cc	24	use PVE::API2::Ceph::MDS;
4fec2764	25	use PVE::API2::Ceph::MGR;
98fe93ae	26	use PVE::API2::Ceph::MON;
79fa41a2	27	use PVE::API2::Ceph::OSD;
3e8560ac DM	28
	29	use PVE::CLIHandler;
	30
	31	use base qw(PVE::CLIHandler);
	32
	33	my $nodename = PVE::INotify::nodename();
	34
	35	my $upid_exit = sub {
	36	my $upid = shift;
	37	my $status = PVE::Tools::upid_read_status($upid);
4bb46baa	38	exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
3e8560ac DM	39	};
3e8560ac DM	40
7e017024 DM	41	sub setup_environment {
	42	PVE::RPCEnvironment->setup_default_cli_env();
	43	}
	44
3e8560ac DM	45	__PACKAGE__->register_method ({
	46	name => 'purge',
	47	path => 'purge',
	48	method => 'POST',
	49	description => "Destroy ceph related data and configuration files.",
	50	parameters => {
	51	additionalProperties => 0,
	52	properties => {
91dfa228 AA	53	logs => {
	54	description => 'Additionally purge Ceph logs, /var/log/ceph.',
	55	type => 'boolean',
	56	optional => 1,
	57	},
	58	crash => {
	59	description => 'Additionally purge Ceph crash logs, /var/lib/ceph/crash.',
	60	type => 'boolean',
	61	optional => 1,
	62	},
3e8560ac DM	63	},
	64	},
	65	returns => { type => 'null' },
	66	code => sub {
	67	my ($param) = @_;
	68
91dfa228 AA	69	my $message;
	70	my $pools = [];
	71	my $monstat = {};
	72	my $mdsstat = {};
	73	my $osdstat = [];
3e8560ac DM	74
	75	eval {
	76	my $rados = PVE::RADOS->new();
91dfa228 AA	77	$pools = PVE::Ceph::Tools::ls_pools(undef, $rados);
	78	$monstat = PVE::Ceph::Services::get_services_info('mon', undef, $rados);
	79	$mdsstat = PVE::Ceph::Services::get_services_info('mds', undef, $rados);
	80	$osdstat = $rados->mon_command({ prefix => 'osd metadata' });
3e8560ac	81	};
c3a3f3ab	82	warn "Error gathering ceph info, already purged? Message: $@" if $@;
3e8560ac	83
91dfa228 AA	84	my $osd = grep { $_->{hostname} eq $nodename } @$osdstat;
	85	my $mds = grep { $mdsstat->{$_}->{host} eq $nodename } keys %$mdsstat;
	86	my $mon = grep { $monstat->{$_}->{host} eq $nodename } keys %$monstat;
3e8560ac	87
91dfa228 AA	88	# no pools = no data
	89	$message .= "- remove pools, this will !!DESTROY DATA!!\n" if @$pools;
	90	$message .= "- remove active OSD on $nodename\n" if $osd;
	91	$message .= "- remove active MDS on $nodename\n" if $mds;
	92	$message .= "- remove other MONs, $nodename is not the last MON\n"
	93	if scalar(keys %$monstat) > 1 && $mon;
	94
	95	# display all steps at once
	96	die "Unable to purge Ceph!\n\nTo continue:\n$message" if $message;
	97
	98	my $services = PVE::Ceph::Services::get_local_services();
	99	$services->{mon} = $monstat if $mon;
	100	$services->{crash}->{$nodename} = { direxists => 1 } if $param->{crash};
	101	$services->{logs}->{$nodename} = { direxists => 1 } if $param->{logs};
	102
	103	PVE::Ceph::Tools::purge_all_ceph_services($services);
	104	PVE::Ceph::Tools::purge_all_ceph_files($services);
3e8560ac DM	105
	106	return undef;
	107	}});
	108
983921b9 TL	109	my $supported_ceph_versions = ['octopus', 'pacific', 'quincy'];
	110	my $default_ceph_version = 'pacific';
	111
3e8560ac DM	112	__PACKAGE__->register_method ({
	113	name => 'install',
	114	path => 'install',
	115	method => 'POST',
	116	description => "Install ceph related packages.",
	117	parameters => {
	118	additionalProperties => 0,
	119	properties => {
	120	version => {
	121	type => 'string',
983921b9 TL	122	enum => $supported_ceph_versions,
983921b9 TL	123	default => $default_ceph_version,
c3d9c698	124	description => "Ceph version to install.",
3e8560ac	125	optional => 1,
61819403 TL	126	},
	127	'allow-experimental' => {
	128	type => 'boolean',
	129	default => 0,
	130	optional => 1,
	131	description => "Allow experimental versions. Use with care!",
	132	},
3f7d054e TL	133	'test-repository' => {
	134	type => 'boolean',
	135	default => 0,
	136	optional => 1,
	137	description => "Use the test, not the main repository. Use with care!",
	138	},
3e8560ac DM	139	},
	140	},
	141	returns => { type => 'null' },
	142	code => sub {
	143	my ($param) = @_;
	144
983921b9	145	my $cephver = $param->{version} \|\| $default_ceph_version;
3e8560ac	146
3f7d054e TL	147	my $repo = $param->{'test-repository'} ? 'test' : 'main';
3f7d054e TL	148
c3d9c698	149	my $repolist;
00fa70bf	150	if ($cephver eq 'octopus') {
ef25743f	151	warn "Ceph Octopus will go EOL after 2022-07\n";
3f7d054e	152	$repolist = "deb http://download.proxmox.com/debian/ceph-octopus bullseye $repo\n";
00fa70bf	153	} elsif ($cephver eq 'pacific') {
3f7d054e	154	$repolist = "deb http://download.proxmox.com/debian/ceph-pacific bullseye $repo\n";
9e81f364 TL	155	} elsif ($cephver eq 'quincy') {
9e81f364 TL	156	$repolist = "deb http://download.proxmox.com/debian/ceph-quincy bullseye $repo\n";
caf37855	157	} else {
3f7d054e	158	die "unsupported ceph version: $cephver";
caf37855	159	}
c3d9c698 TL	160	PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $repolist);
c3d9c698 TL	161
7271e6f6 TL	162	my $supported_re = join('\|', $supported_ceph_versions->@*);
7271e6f6 TL	163	warn "WARNING: installing non-default ceph release '$cephver'!\n" if $cephver !~ qr/^(?:$supported_re)$/;
3e8560ac	164
87166f36	165	local $ENV{DEBIAN_FRONTEND} = 'noninteractive';
3e8560ac	166	print "update available package list\n";
d26556c0 TL	167	eval {
	168	run_command(
	169	['apt-get', '-q', 'update'],
	170	outfunc => sub {},
	171	errfunc => sub { print STDERR "$_[0]\n" },
	172	)
	173	};
3e8560ac	174
ecddd2e2	175	my @apt_install = qw(apt-get --no-install-recommends -o Dpkg::Options::=--force-confnew install --);
2ca75379 TL	176	my @ceph_packages = qw(
	177	ceph
	178	ceph-common
	179	ceph-mds
	180	ceph-fuse
	181	gdisk
08e22c1e	182	nvme-cli
2ca75379 TL	183	);
2ca75379 TL	184
b23305d5 TL	185	# got split out with quincy and is required by PVE tooling, conditionally exclude it for older
b23305d5 TL	186	# FIXME: remove condition with PVE 8.0, i.e., once we only support quincy+ new installations
3409fbea SS	187	if ($cephver ne 'octopus' and $cephver ne 'pacific') {
	188	push @ceph_packages, 'ceph-volume';
	189	}
	190
55ab726e	191	print "start installation\n";
4dd27d50	192
d380d000	193	# this flag helps to determine when apt is actually done installing (vs. partial extracing)
9f6dc075	194	my $install_flag_fn = PVE::Ceph::Tools::ceph_install_flag_file();
d380d000	195	open(my $install_flag, '>', $install_flag_fn) or die "could not create install flag - $!\n";
4dd27d50 AL	196	close $install_flag;
4dd27d50 AL	197
2ca75379	198	if (system(@apt_install, @ceph_packages) != 0) {
d380d000	199	unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
2ca75379 TL	200	die "apt failed during ceph installation ($?)\n";
2ca75379 TL	201	}
55ab726e	202
37bf860e	203	print "\ninstalled ceph $cephver successfully!\n";
d380d000 TL	204	# done: drop flag file so that the PVE::Ceph::Tools check returns Ok now.
d380d000 TL	205	unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
ece49b3c TL	206
	207	print "\nreloading API to load new Ceph RADOS library...\n";
	208	run_command([
	209	'systemctl', 'try-reload-or-restart', 'pvedaemon.service', 'pveproxy.service'
	210	]);
3e8560ac DM	211
	212	return undef;
	213	}});
	214
1baeb2d1 AL	215	__PACKAGE__->register_method ({
	216	name => 'status',
	217	path => 'status',
	218	method => 'GET',
	219	description => "Get Ceph Status.",
	220	parameters => {
	221	additionalProperties => 0,
	222	},
	223	returns => { type => 'null' },
	224	code => sub {
	225	PVE::Ceph::Tools::check_ceph_inited();
	226
	227	run_command(
	228	['ceph', '-s'],
	229	outfunc => sub { print "$_[0]\n" },
ab459f6d TL	230	errfunc => sub { print STDERR "$_[0]\n" },
ab459f6d TL	231	timeout => 15,
1baeb2d1 AL	232	);
	233	return undef;
	234	}});
	235
02c1e98e DC	236	my $get_storages = sub {
	237	my ($fs, $is_default) = @_;
	238
	239	my $cfg = PVE::Storage::config();
	240
	241	my $storages = $cfg->{ids};
	242	my $res = {};
	243	foreach my $storeid (keys %$storages) {
	244	my $curr = $storages->{$storeid};
	245	next if $curr->{type} ne 'cephfs';
	246	my $cur_fs = $curr->{'fs-name'};
	247	$res->{$storeid} = $storages->{$storeid}
	248	if (!defined($cur_fs) && $is_default) \|\| (defined($cur_fs) && $fs eq $cur_fs);
	249	}
	250
	251	return $res;
	252	};
	253
	254	__PACKAGE__->register_method ({
	255	name => 'destroyfs',
	256	path => 'destroyfs',
	257	method => 'DELETE',
	258	description => "Destroy a Ceph filesystem",
	259	parameters => {
	260	additionalProperties => 0,
	261	properties => {
	262	node => get_standard_option('pve-node'),
	263	name => {
	264	description => "The ceph filesystem name.",
	265	type => 'string',
	266	},
	267	'remove-storages' => {
	268	description => "Remove all pveceph-managed storages configured for this fs.",
	269	type => 'boolean',
	270	optional => 1,
	271	default => 0,
	272	},
	273	'remove-pools' => {
	274	description => "Remove data and metadata pools configured for this fs.",
	275	type => 'boolean',
	276	optional => 1,
	277	default => 0,
	278	},
	279	},
	280	},
	281	returns => { type => 'string' },
	282	code => sub {
	283	my ($param) = @_;
	284
	285	PVE::Ceph::Tools::check_ceph_inited();
	286
	287	my $rpcenv = PVE::RPCEnvironment::get();
	288	my $user = $rpcenv->get_user();
	289
	290	my $fs_name = $param->{name};
	291
	292	my $fs;
	293	my $fs_list = PVE::Ceph::Tools::ls_fs();
	294	for my $entry (@$fs_list) {
	295	next if $entry->{name} ne $fs_name;
	296	$fs = $entry;
	297	last;
	298	}
	299	die "no such cephfs '$fs_name'\n" if !$fs;
300
301	my $worker = sub {
302	my $rados = PVE::RADOS->new();
303
304	if ($param->{'remove-storages'}) {
305	my $defaultfs;
306	my $fs_dump = $rados->mon_command({ prefix => "fs dump" });
307	for my $fs ($fs_dump->{filesystems}->@*) {
308	next if $fs->{id} != $fs_dump->{default_fscid};
309	$defaultfs = $fs->{mdsmap}->{fs_name};
310	}
311	warn "no default fs found, maybe not all relevant storages are removed\n"
312	if !defined($defaultfs);
313
314	my $storages = $get_storages->($fs_name, $fs_name eq ($defaultfs // ''));
315	for my $storeid (keys %$storages) {
316	my $store = $storages->{$storeid};
317	if (!$store->{disable}) {
318	die "storage '$storeid' is not disabled, make sure to disable ".
319	"and unmount the storage first\n";
320	}
321	}
322
323	my $err;
324	for my $storeid (keys %$storages) {
325	# skip external clusters, not managed by pveceph
326	next if $storages->{$storeid}->{monhost};
327	eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
328	if ($@) {
329	warn "failed to remove storage '$storeid': $@\n";
330	$err = 1;
331	}
332	}
333	die "failed to remove (some) storages - check log and remove manually!\n"
334	if $err;
335	}
336
337	PVE::Ceph::Tools::destroy_fs($fs_name, $rados);
338
339	if ($param->{'remove-pools'}) {
340	warn "removing metadata pool '$fs->{metadata_pool}'\n";
341	eval { PVE::Ceph::Tools::destroy_pool($fs->{metadata_pool}, $rados) };
342	warn "$@\n" if $@;
343
344	foreach my $pool ($fs->{data_pools}->@*) {
345	warn "removing data pool '$pool'\n";
346	eval { PVE::Ceph::Tools::destroy_pool($pool, $rados) };
347	warn "$@\n" if $@;
348	}
349	}
350
351	};
352	return $rpcenv->fork_worker('cephdestroyfs', $fs_name, $user, $worker);
353	}});
354
3e8560ac DM	355	our $cmddef = {
3e8560ac DM	356	init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ],
8c476782	357	pool => {
a0665001	358	ls => [ 'PVE::API2::Ceph::Pool', 'lspools', [], { node => $nodename }, sub {
d4dba076 AA	359	my ($data, $schema, $options) = @_;
	360	PVE::CLIFormatter::print_api_result($data, $schema,
	361	[
	362	'pool_name',
	363	'size',
	364	'min_size',
	365	'pg_num',
5a3d7942 AA	366	'pg_num_min',
5a3d7942 AA	367	'pg_num_final',
d4dba076	368	'pg_autoscale_mode',
5a3d7942 AA	369	'target_size',
5a3d7942 AA	370	'target_size_ratio',
d4dba076 AA	371	'crush_rule_name',
	372	'percent_used',
	373	'bytes_used',
	374	],
	375	$options);
	376	}, $PVE::RESTHandler::standard_output_options],
a0665001 AL	377	create => [ 'PVE::API2::Ceph::Pool', 'createpool', ['name'], { node => $nodename }],
	378	destroy => [ 'PVE::API2::Ceph::Pool', 'destroypool', ['name'], { node => $nodename } ],
	379	set => [ 'PVE::API2::Ceph::Pool', 'setpool', ['name'], { node => $nodename } ],
	380	get => [ 'PVE::API2::Ceph::Pool', 'getpool', ['name'], { node => $nodename }, sub {
54ba7dd9 AA	381	my ($data, $schema, $options) = @_;
	382	PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
	383	}, $PVE::RESTHandler::standard_output_options],
8c476782 TL	384	},
	385	lspools => { alias => 'pool ls' },
	386	createpool => { alias => 'pool create' },
	387	destroypool => { alias => 'pool destroy' },
	388	fs => {
	389	create => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
02c1e98e	390	destroy => [ __PACKAGE__, 'destroyfs', ['name'], { node => $nodename }],
8c476782 TL	391	},
8c476782 TL	392	osd => {
79fa41a2 DC	393	create => [ 'PVE::API2::Ceph::OSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
79fa41a2 DC	394	destroy => [ 'PVE::API2::Ceph::OSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
8c476782 TL	395	},
	396	createosd => { alias => 'osd create' },
	397	destroyosd => { alias => 'osd destroy' },
	398	mon => {
98fe93ae DC	399	create => [ 'PVE::API2::Ceph::MON', 'createmon', [], { node => $nodename }, $upid_exit],
98fe93ae DC	400	destroy => [ 'PVE::API2::Ceph::MON', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
8c476782 TL	401	},
	402	createmon => { alias => 'mon create' },
	403	destroymon => { alias => 'mon destroy' },
	404	mgr => {
4fec2764 DC	405	create => [ 'PVE::API2::Ceph::MGR', 'createmgr', [], { node => $nodename }, $upid_exit],
4fec2764 DC	406	destroy => [ 'PVE::API2::Ceph::MGR', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
8c476782 TL	407	},
	408	createmgr => { alias => 'mgr create' },
	409	destroymgr => { alias => 'mgr destroy' },
	410	mds => {
	411	create => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
f16bb531	412	destroy => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['name'], { node => $nodename }, $upid_exit],
8c476782	413	},
7da6ff26 DJ	414	start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename }, $upid_exit],
7da6ff26 DJ	415	stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename }, $upid_exit],
3e8560ac DM	416	install => [ __PACKAGE__, 'install', [] ],
3e8560ac DM	417	purge => [ __PACKAGE__, 'purge', [] ],
1baeb2d1	418	status => [ __PACKAGE__, 'status', []],
3e8560ac DM	419	};
	420
	421	1;