[pve-manager.git] / PVE / CLI / pveceph.pm

package PVE::CLI::pveceph;

use strict;
use warnings;

use Fcntl ':flock';
use File::Path;
use IO::File;
use JSON;
use Data::Dumper;
use LWP::UserAgent;

use Proxmox::RS::Subscription;

use PVE::SafeSyslog;
use PVE::Cluster;
use PVE::INotify;
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::Tools qw(run_command);
use PVE::JSONSchema qw(get_standard_option);
use PVE::Ceph::Tools;
use PVE::Ceph::Services;
use PVE::API2::Ceph;
use PVE::API2::Ceph::FS;
use PVE::API2::Ceph::MDS;
use PVE::API2::Ceph::MGR;
use PVE::API2::Ceph::MON;
use PVE::API2::Ceph::OSD;

use PVE::CLIHandler;

use base qw(PVE::CLIHandler);

my $nodename = PVE::INotify::nodename();

my $upid_exit = sub {
    my $upid = shift;
    my $status = PVE::Tools::upid_read_status($upid);
    exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
};

sub setup_environment {
    PVE::RPCEnvironment->setup_default_cli_env();
}

__PACKAGE__->register_method ({
    name => 'purge',
    path => 'purge',
    method => 'POST',
    description => "Destroy ceph related data and configuration files.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    logs => {
		description => 'Additionally purge Ceph logs, /var/log/ceph.',
		type => 'boolean',
		optional => 1,
	    },
	    crash => {
		description => 'Additionally purge Ceph crash logs, /var/lib/ceph/crash.',
		type => 'boolean',
		optional => 1,
	    },
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	my $message;
	my $pools = [];
	my $monstat = {};
	my $mdsstat = {};
	my $osdstat = [];

	eval {
	    my $rados = PVE::RADOS->new();
	    $pools = PVE::Ceph::Tools::ls_pools(undef, $rados);
	    $monstat = PVE::Ceph::Services::get_services_info('mon', undef, $rados);
	    $mdsstat = PVE::Ceph::Services::get_services_info('mds', undef, $rados);
	    $osdstat = $rados->mon_command({ prefix => 'osd metadata' });
	};
	warn "Error gathering ceph info, already purged? Message: $@" if $@;

	my $osd = grep { $_->{hostname} eq $nodename } @$osdstat;
	my $mds = grep { $mdsstat->{$_}->{host} eq $nodename } keys %$mdsstat;
	my $mon = grep { $monstat->{$_}->{host} eq $nodename } keys %$monstat;

	# no pools = no data
	$message .= "- remove pools, this will !!DESTROY DATA!!\n" if @$pools;
	$message .= "- remove active OSD on $nodename\n" if $osd;
	$message .= "- remove active MDS on $nodename\n" if $mds;
	$message .= "- remove other MONs, $nodename is not the last MON\n"
	    if scalar(keys %$monstat) > 1 && $mon;

	# display all steps at once
	die "Unable to purge Ceph!\n\nTo continue:\n$message" if $message;

	my $services = PVE::Ceph::Services::get_local_services();
	$services->{mon} = $monstat if $mon;
	$services->{crash}->{$nodename} = { direxists => 1 } if $param->{crash};
	$services->{logs}->{$nodename} = { direxists => 1 } if $param->{logs};

	PVE::Ceph::Tools::purge_all_ceph_services($services);
	PVE::Ceph::Tools::purge_all_ceph_files($services);

	return undef;
    }});

my sub has_valid_subscription {
    my $info = eval { Proxmox::RS::Subscription::read_subscription('/etc/subscription') } // {};
    warn "couldn't check subscription info - $@" if $@;
    return $info->{status} && $info->{status} eq 'active'; # age check?
}

my $supported_ceph_versions = ['quincy'];
my $default_ceph_version = 'quincy';

__PACKAGE__->register_method ({
    name => 'install',
    path => 'install',
    method => 'POST',
    description => "Install ceph related packages.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    version => {
		type => 'string',
		enum => $supported_ceph_versions,
		default => $default_ceph_version,
		description => "Ceph version to install.",
		optional => 1,
	    },
	    repository => {
		type => 'string',
		enum => ['enterprise', 'no-subscription', 'test'],
		default => 'enterprise',
		description => "Ceph repository to use.",
		optional => 1,
	    },
	    'allow-experimental' => {
		type => 'boolean',
		default => 0,
		optional => 1,
		description => "Allow experimental versions. Use with care!",
	    },
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	my $cephver = $param->{version} || $default_ceph_version;

	my $repo = $param->{'repository'} // 'enterprise';
	my $enterprise_repo = $repo eq 'enterprise';
	my $cdn = $enterprise_repo ? 'https://enterprise.proxmox.com' : 'http://download.proxmox.com';

	if (has_valid_subscription()) {
	    warn "\nNOTE: The node has an active subscription but a non-production Ceph repository selected.\n\n"
	        if !$enterprise_repo;
	} elsif ($enterprise_repo) {
	    warn "\nWARN: Enterprise repository selected, but no active subscription!\n\n";
	} elsif ($repo eq 'no-subscription') {
	    warn "\nHINT: The no-subscription repository is not the best choice for production setups.\n"
	        ."Proxmox recommends using the enterprise repository with a valid subscription.\n";
	} else {
	    warn "\nWARN: The test repository should only be used for test setups or after consulting"
		." the official Proxmox support!\n\n"
	}

	my $repolist;
	if ($cephver eq 'quincy') {
	    $repolist = "deb ${cdn}/debian/ceph-quincy bookworm $repo\n";
	} else {
	    die "unsupported ceph version: $cephver";
	}

	if (-t STDOUT && !$param->{version}) {
	    print "This will install Ceph " . ucfirst($cephver) . " - continue (y/N)? ";

	    my $answer = <STDIN>;
	    my $continue = defined($answer) && $answer =~ m/^\s*y(?:es)?\s*$/i;

	    die "Aborting installation as requested\n" if !$continue;
	}

	PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $repolist);

	my $supported_re = join('|', $supported_ceph_versions->@*);
	warn "WARNING: installing non-default ceph release '$cephver'!\n" if $cephver !~ qr/^(?:$supported_re)$/;

	local $ENV{DEBIAN_FRONTEND} = 'noninteractive';
	print "update available package list\n";
	eval {
	    run_command(
		['apt-get', '-q', 'update'],
		outfunc => sub {},
		errfunc => sub { print STDERR "$_[0]\n" },
	    )
	};

	my @apt_install = qw(apt-get --no-install-recommends -o Dpkg::Options::=--force-confnew install --);
	my @ceph_packages = qw(
	    ceph
	    ceph-common
	    ceph-fuse
	    ceph-mds
	    ceph-volume
	    gdisk
	    nvme-cli
	);

	print "start installation\n";

	# this flag helps to determine when apt is actually done installing (vs. partial extracing)
	my $install_flag_fn = PVE::Ceph::Tools::ceph_install_flag_file();
	open(my $install_flag, '>', $install_flag_fn) or die "could not create install flag - $!\n";
	close $install_flag;

	if (system(@apt_install, @ceph_packages) != 0) {
	    unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
	    die "apt failed during ceph installation ($?)\n";
	}

	print "\ninstalled ceph $cephver successfully!\n";
	# done: drop flag file so that the PVE::Ceph::Tools check returns Ok now.
	unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";

	print "\nreloading API to load new Ceph RADOS library...\n";
	run_command([
	    'systemctl', 'try-reload-or-restart', 'pvedaemon.service', 'pveproxy.service'
	]);

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'status',
    path => 'status',
    method => 'GET',
    description => "Get Ceph Status.",
    parameters => {
	additionalProperties => 0,
    },
    returns => { type => 'null' },
    code => sub {
	PVE::Ceph::Tools::check_ceph_inited();

	run_command(
	    ['ceph', '-s'],
	    outfunc => sub { print "$_[0]\n" },
	    errfunc => sub { print STDERR "$_[0]\n" },
	    timeout => 15,
	);
	return undef;
    }});

my $get_storages = sub {
    my ($fs, $is_default) = @_;

    my $cfg = PVE::Storage::config();

    my $storages = $cfg->{ids};
    my $res = {};
    foreach my $storeid (keys %$storages) {
	my $curr = $storages->{$storeid};
	next if $curr->{type} ne 'cephfs';
	my $cur_fs = $curr->{'fs-name'};
	$res->{$storeid} = $storages->{$storeid}
	    if (!defined($cur_fs) && $is_default) || (defined($cur_fs) && $fs eq $cur_fs);
    }

    return $res;
};

__PACKAGE__->register_method ({
    name => 'destroyfs',
    path => 'destroyfs',
    method => 'DELETE',
    description => "Destroy a Ceph filesystem",
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    name => {
		description => "The ceph filesystem name.",
		type => 'string',
	    },
	    'remove-storages' => {
		description => "Remove all pveceph-managed storages configured for this fs.",
		type => 'boolean',
		optional => 1,
		default => 0,
	    },
	    'remove-pools' => {
		description => "Remove data and metadata pools configured for this fs.",
		type => 'boolean',
		optional => 1,
		default => 0,
	    },
	},
    },
    returns => { type => 'string' },
    code => sub {
	my ($param) = @_;

	PVE::Ceph::Tools::check_ceph_inited();

	my $rpcenv = PVE::RPCEnvironment::get();
	my $user = $rpcenv->get_user();

	my $fs_name = $param->{name};

	my $fs;
	my $fs_list = PVE::Ceph::Tools::ls_fs();
	for my $entry (@$fs_list) {
	    next if $entry->{name} ne $fs_name;
	    $fs = $entry;
	    last;
	}
	die "no such cephfs '$fs_name'\n" if !$fs;

	my $worker = sub {
	    my $rados = PVE::RADOS->new();

	    if ($param->{'remove-storages'}) {
		my $defaultfs;
		my $fs_dump = $rados->mon_command({ prefix => "fs dump" });
		for my $fs ($fs_dump->{filesystems}->@*) {
		    next if $fs->{id} != $fs_dump->{default_fscid};
		    $defaultfs = $fs->{mdsmap}->{fs_name};
		}
		warn "no default fs found, maybe not all relevant storages are removed\n"
		    if !defined($defaultfs);

		my $storages = $get_storages->($fs_name, $fs_name eq ($defaultfs // ''));
		for my $storeid (keys %$storages) {
		    my $store = $storages->{$storeid};
		    if (!$store->{disable}) {
			die "storage '$storeid' is not disabled, make sure to disable ".
			    "and unmount the storage first\n";
		    }
		}

		my $err;
		for my $storeid (keys %$storages) {
		    # skip external clusters, not managed by pveceph
		    next if $storages->{$storeid}->{monhost};
		    eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
		    if ($@) {
			warn "failed to remove storage '$storeid': $@\n";
			$err = 1;
		    }
		}
		die "failed to remove (some) storages - check log and remove manually!\n"
		    if $err;
	    }

	    PVE::Ceph::Tools::destroy_fs($fs_name, $rados);

	    if ($param->{'remove-pools'}) {
		warn "removing metadata pool '$fs->{metadata_pool}'\n";
		eval { PVE::Ceph::Tools::destroy_pool($fs->{metadata_pool}, $rados) };
		warn "$@\n" if $@;

		foreach my $pool ($fs->{data_pools}->@*) {
		    warn "removing data pool '$pool'\n";
		    eval { PVE::Ceph::Tools::destroy_pool($pool, $rados) };
		    warn "$@\n" if $@;
		}
	    }

	};
	return $rpcenv->fork_worker('cephdestroyfs', $fs_name,  $user, $worker);
    }});

__PACKAGE__->register_method ({
    name => 'osd-details',
    path => 'osd-details',
    method => 'GET',
    description => "Get OSD details.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    osdid => {
		description => "ID of the OSD",
		type => 'string',
	    },
	    verbose => {
		description => "Print verbose information, same as json-pretty output format.",
		type => 'boolean',
		default => 0,
		optional => 1,
	    },
	},
    },
    returns => { type => 'object' },
    code => sub {
	my ($param) = @_;

	PVE::Ceph::Tools::check_ceph_inited();

	my $res = PVE::API2::Ceph::OSD->osddetails({
	    osdid => $param->{osdid},
	    node => $param->{node},
	});

	for my $dev ($res->{devices}->@*) {
	    $dev->{"lv-info"} = PVE::API2::Ceph::OSD->osdvolume({
		osdid => $param->{osdid},
		node => $param->{node},
		type => $dev->{device},
	    });
	}
	$res->{verbose} = 1 if $param->{verbose};
	return $res;
    }});

my $format_osddetails = sub {
    my ($data, $schema, $options) = @_;

    $options->{"output-format"} //= "text";

    if ($data->{verbose}) {
	$options->{"output-format"} = "json-pretty";
	delete $data->{verbose};
    }

    if ($options->{"output-format"} eq "text") {
	for my $dev ($data->{devices}->@*) {
	    my ($disk, $type, $device) = $dev->@{'physical_device', 'type', 'device'};
	    my ($lv_size, $lv_ctime) = $dev->{'lv-info'}->@{'lv_size', 'creation_time'};

	    $data->{osd}->{$device} = "Disk: $disk, Type: $type, LV Size: $lv_size, LV Creation Time: $lv_ctime";
	}
	PVE::CLIFormatter::print_api_result($data->{osd}, $schema, undef, $options);
    } else {
	PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
    }
};

our $cmddef = {
    init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ],
    pool => {
	ls => [ 'PVE::API2::Ceph::Pool', 'lspools', [], { node => $nodename }, sub {
	    my ($data, $schema, $options) = @_;
	    PVE::CLIFormatter::print_api_result($data, $schema,
		[
		    'pool_name',
		    'size',
		    'min_size',
		    'pg_num',
		    'pg_num_min',
		    'pg_num_final',
		    'pg_autoscale_mode',
		    'target_size',
		    'target_size_ratio',
		    'crush_rule_name',
		    'percent_used',
		    'bytes_used',
		],
		$options);
	}, $PVE::RESTHandler::standard_output_options],
	create => [ 'PVE::API2::Ceph::Pool', 'createpool', ['name'], { node => $nodename }],
	destroy => [ 'PVE::API2::Ceph::Pool', 'destroypool', ['name'], { node => $nodename } ],
	set => [ 'PVE::API2::Ceph::Pool', 'setpool', ['name'], { node => $nodename } ],
	get => [ 'PVE::API2::Ceph::Pool', 'getpool', ['name'], { node => $nodename }, sub {
	    my ($data, $schema, $options) = @_;
	    PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
	}, $PVE::RESTHandler::standard_output_options],
    },
    lspools => { alias => 'pool ls' },
    createpool => { alias => 'pool create' },
    destroypool => { alias => 'pool destroy' },
    fs => {
	create => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
	destroy => [ __PACKAGE__, 'destroyfs', ['name'], { node => $nodename }],
    },
    osd => {
	create => [ 'PVE::API2::Ceph::OSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
	destroy => [ 'PVE::API2::Ceph::OSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
	details => [
	    __PACKAGE__, 'osd-details', ['osdid'], { node => $nodename }, $format_osddetails,
	    $PVE::RESTHandler::standard_output_options,
	],
    },
    createosd => { alias => 'osd create' },
    destroyosd => { alias => 'osd destroy' },
    mon => {
	create => [ 'PVE::API2::Ceph::MON', 'createmon', [], { node => $nodename }, $upid_exit],
	destroy => [ 'PVE::API2::Ceph::MON', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
    },
    createmon => { alias => 'mon create' },
    destroymon => { alias => 'mon destroy' },
    mgr => {
	create => [ 'PVE::API2::Ceph::MGR', 'createmgr', [], { node => $nodename }, $upid_exit],
	destroy => [ 'PVE::API2::Ceph::MGR', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
    },
    createmgr => { alias => 'mgr create' },
    destroymgr => { alias => 'mgr destroy' },
    mds => {
	create => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
	destroy => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['name'], { node => $nodename }, $upid_exit],
    },
    start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename }, $upid_exit],
    stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename }, $upid_exit],
    install => [ __PACKAGE__, 'install', [] ],
    purge => [  __PACKAGE__, 'purge', [] ],
    status => [ __PACKAGE__, 'status', []],
};

1;
Commit	Line	Data
3e8560ac DM	1	package PVE::CLI::pveceph;
	2
	3	use strict;
	4	use warnings;
	5
	6	use Fcntl ':flock';
	7	use File::Path;
	8	use IO::File;
	9	use JSON;
	10	use Data::Dumper;
	11	use LWP::UserAgent;
	12
bbfbf0ef TL	13	use Proxmox::RS::Subscription;
bbfbf0ef TL	14
3e8560ac DM	15	use PVE::SafeSyslog;
	16	use PVE::Cluster;
	17	use PVE::INotify;
	18	use PVE::RPCEnvironment;
	19	use PVE::Storage;
	20	use PVE::Tools qw(run_command);
	21	use PVE::JSONSchema qw(get_standard_option);
6fb08cb9	22	use PVE::Ceph::Tools;
91dfa228	23	use PVE::Ceph::Services;
3e8560ac	24	use PVE::API2::Ceph;
7e1a9d25	25	use PVE::API2::Ceph::FS;
b82649cc	26	use PVE::API2::Ceph::MDS;
4fec2764	27	use PVE::API2::Ceph::MGR;
98fe93ae	28	use PVE::API2::Ceph::MON;
79fa41a2	29	use PVE::API2::Ceph::OSD;
3e8560ac DM	30
	31	use PVE::CLIHandler;
	32
	33	use base qw(PVE::CLIHandler);
	34
	35	my $nodename = PVE::INotify::nodename();
	36
	37	my $upid_exit = sub {
	38	my $upid = shift;
	39	my $status = PVE::Tools::upid_read_status($upid);
4bb46baa	40	exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
3e8560ac DM	41	};
3e8560ac DM	42
7e017024 DM	43	sub setup_environment {
	44	PVE::RPCEnvironment->setup_default_cli_env();
	45	}
	46
3e8560ac DM	47	__PACKAGE__->register_method ({
	48	name => 'purge',
	49	path => 'purge',
	50	method => 'POST',
	51	description => "Destroy ceph related data and configuration files.",
	52	parameters => {
	53	additionalProperties => 0,
	54	properties => {
91dfa228 AA	55	logs => {
	56	description => 'Additionally purge Ceph logs, /var/log/ceph.',
	57	type => 'boolean',
	58	optional => 1,
	59	},
	60	crash => {
	61	description => 'Additionally purge Ceph crash logs, /var/lib/ceph/crash.',
	62	type => 'boolean',
	63	optional => 1,
	64	},
3e8560ac DM	65	},
	66	},
	67	returns => { type => 'null' },
	68	code => sub {
	69	my ($param) = @_;
	70
91dfa228 AA	71	my $message;
	72	my $pools = [];
	73	my $monstat = {};
	74	my $mdsstat = {};
	75	my $osdstat = [];
3e8560ac DM	76
	77	eval {
	78	my $rados = PVE::RADOS->new();
91dfa228 AA	79	$pools = PVE::Ceph::Tools::ls_pools(undef, $rados);
	80	$monstat = PVE::Ceph::Services::get_services_info('mon', undef, $rados);
	81	$mdsstat = PVE::Ceph::Services::get_services_info('mds', undef, $rados);
	82	$osdstat = $rados->mon_command({ prefix => 'osd metadata' });
3e8560ac	83	};
c3a3f3ab	84	warn "Error gathering ceph info, already purged? Message: $@" if $@;
3e8560ac	85
91dfa228 AA	86	my $osd = grep { $_->{hostname} eq $nodename } @$osdstat;
	87	my $mds = grep { $mdsstat->{$_}->{host} eq $nodename } keys %$mdsstat;
	88	my $mon = grep { $monstat->{$_}->{host} eq $nodename } keys %$monstat;
3e8560ac	89
91dfa228 AA	90	# no pools = no data
	91	$message .= "- remove pools, this will !!DESTROY DATA!!\n" if @$pools;
	92	$message .= "- remove active OSD on $nodename\n" if $osd;
	93	$message .= "- remove active MDS on $nodename\n" if $mds;
	94	$message .= "- remove other MONs, $nodename is not the last MON\n"
	95	if scalar(keys %$monstat) > 1 && $mon;
	96
	97	# display all steps at once
	98	die "Unable to purge Ceph!\n\nTo continue:\n$message" if $message;
	99
	100	my $services = PVE::Ceph::Services::get_local_services();
	101	$services->{mon} = $monstat if $mon;
	102	$services->{crash}->{$nodename} = { direxists => 1 } if $param->{crash};
	103	$services->{logs}->{$nodename} = { direxists => 1 } if $param->{logs};
	104
	105	PVE::Ceph::Tools::purge_all_ceph_services($services);
	106	PVE::Ceph::Tools::purge_all_ceph_files($services);
3e8560ac DM	107
	108	return undef;
	109	}});
	110
bbfbf0ef TL	111	my sub has_valid_subscription {
	112	my $info = eval { Proxmox::RS::Subscription::read_subscription('/etc/subscription') } // {};
	113	warn "couldn't check subscription info - $@" if $@;
	114	return $info->{status} && $info->{status} eq 'active'; # age check?
	115	}
	116
74ea1fb2 TL	117	my $supported_ceph_versions = ['quincy'];
74ea1fb2 TL	118	my $default_ceph_version = 'quincy';
983921b9	119
3e8560ac DM	120	__PACKAGE__->register_method ({
	121	name => 'install',
	122	path => 'install',
	123	method => 'POST',
	124	description => "Install ceph related packages.",
	125	parameters => {
	126	additionalProperties => 0,
	127	properties => {
	128	version => {
	129	type => 'string',
983921b9 TL	130	enum => $supported_ceph_versions,
983921b9 TL	131	default => $default_ceph_version,
c3d9c698	132	description => "Ceph version to install.",
3e8560ac	133	optional => 1,
61819403	134	},
93542d77 TL	135	repository => {
	136	type => 'string',
	137	enum => ['enterprise', 'no-subscription', 'test'],
	138	default => 'enterprise',
	139	description => "Ceph repository to use.",
61819403	140	optional => 1,
61819403	141	},
93542d77	142	'allow-experimental' => {
3f7d054e TL	143	type => 'boolean',
	144	default => 0,
	145	optional => 1,
93542d77	146	description => "Allow experimental versions. Use with care!",
3f7d054e	147	},
3e8560ac DM	148	},
	149	},
	150	returns => { type => 'null' },
	151	code => sub {
	152	my ($param) = @_;
	153
983921b9	154	my $cephver = $param->{version} \|\| $default_ceph_version;
3e8560ac	155
93542d77 TL	156	my $repo = $param->{'repository'} // 'enterprise';
	157	my $enterprise_repo = $repo eq 'enterprise';
	158	my $cdn = $enterprise_repo ? 'https://enterprise.proxmox.com' : 'http://download.proxmox.com';
3f7d054e	159
bbfbf0ef TL	160	if (has_valid_subscription()) {
	161	warn "\nNOTE: The node has an active subscription but a non-production Ceph repository selected.\n\n"
	162	if !$enterprise_repo;
	163	} elsif ($enterprise_repo) {
	164	warn "\nWARN: Enterprise repository selected, but no active subscription!\n\n";
	165	} elsif ($repo eq 'no-subscription') {
	166	warn "\nHINT: The no-subscription repository is not the best choice for production setups.\n"
	167	."Proxmox recommends using the enterprise repository with a valid subscription.\n";
	168	} else {
	169	warn "\nWARN: The test repository should only be used for test setups or after consulting"
	170	." the official Proxmox support!\n\n"
	171	}
	172
c3d9c698	173	my $repolist;
74ea1fb2	174	if ($cephver eq 'quincy') {
93542d77	175	$repolist = "deb ${cdn}/debian/ceph-quincy bookworm $repo\n";
caf37855	176	} else {
3f7d054e	177	die "unsupported ceph version: $cephver";
caf37855	178	}
d1c49f5b MC	179
	180	if (-t STDOUT && !$param->{version}) {
	181	print "This will install Ceph " . ucfirst($cephver) . " - continue (y/N)? ";
	182
	183	my $answer = <STDIN>;
	184	my $continue = defined($answer) && $answer =~ m/^\sy(?:es)?\s$/i;
	185
	186	die "Aborting installation as requested\n" if !$continue;
	187	}
	188
c3d9c698 TL	189	PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $repolist);
c3d9c698 TL	190
7271e6f6 TL	191	my $supported_re = join('\|', $supported_ceph_versions->@*);
7271e6f6 TL	192	warn "WARNING: installing non-default ceph release '$cephver'!\n" if $cephver !~ qr/^(?:$supported_re)$/;
3e8560ac	193
87166f36	194	local $ENV{DEBIAN_FRONTEND} = 'noninteractive';
3e8560ac	195	print "update available package list\n";
d26556c0 TL	196	eval {
	197	run_command(
	198	['apt-get', '-q', 'update'],
	199	outfunc => sub {},
	200	errfunc => sub { print STDERR "$_[0]\n" },
	201	)
	202	};
3e8560ac	203
ecddd2e2	204	my @apt_install = qw(apt-get --no-install-recommends -o Dpkg::Options::=--force-confnew install --);
2ca75379 TL	205	my @ceph_packages = qw(
	206	ceph
	207	ceph-common
2ca75379	208	ceph-fuse
ddd89279 TL	209	ceph-mds
ddd89279 TL	210	ceph-volume
2ca75379	211	gdisk
08e22c1e	212	nvme-cli
2ca75379 TL	213	);
2ca75379 TL	214
55ab726e	215	print "start installation\n";
4dd27d50	216
d380d000	217	# this flag helps to determine when apt is actually done installing (vs. partial extracing)
9f6dc075	218	my $install_flag_fn = PVE::Ceph::Tools::ceph_install_flag_file();
d380d000	219	open(my $install_flag, '>', $install_flag_fn) or die "could not create install flag - $!\n";
4dd27d50 AL	220	close $install_flag;
4dd27d50 AL	221
2ca75379	222	if (system(@apt_install, @ceph_packages) != 0) {
d380d000	223	unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
2ca75379 TL	224	die "apt failed during ceph installation ($?)\n";
2ca75379 TL	225	}
55ab726e	226
37bf860e	227	print "\ninstalled ceph $cephver successfully!\n";
d380d000 TL	228	# done: drop flag file so that the PVE::Ceph::Tools check returns Ok now.
d380d000 TL	229	unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
ece49b3c TL	230
	231	print "\nreloading API to load new Ceph RADOS library...\n";
	232	run_command([
	233	'systemctl', 'try-reload-or-restart', 'pvedaemon.service', 'pveproxy.service'
	234	]);
3e8560ac DM	235
	236	return undef;
	237	}});
	238
1baeb2d1 AL	239	__PACKAGE__->register_method ({
	240	name => 'status',
	241	path => 'status',
	242	method => 'GET',
	243	description => "Get Ceph Status.",
	244	parameters => {
	245	additionalProperties => 0,
	246	},
	247	returns => { type => 'null' },
	248	code => sub {
	249	PVE::Ceph::Tools::check_ceph_inited();
	250
	251	run_command(
	252	['ceph', '-s'],
	253	outfunc => sub { print "$_[0]\n" },
ab459f6d TL	254	errfunc => sub { print STDERR "$_[0]\n" },
ab459f6d TL	255	timeout => 15,
1baeb2d1 AL	256	);
	257	return undef;
	258	}});
	259
02c1e98e DC	260	my $get_storages = sub {
	261	my ($fs, $is_default) = @_;
	262
	263	my $cfg = PVE::Storage::config();
	264
	265	my $storages = $cfg->{ids};
	266	my $res = {};
	267	foreach my $storeid (keys %$storages) {
	268	my $curr = $storages->{$storeid};
	269	next if $curr->{type} ne 'cephfs';
	270	my $cur_fs = $curr->{'fs-name'};
	271	$res->{$storeid} = $storages->{$storeid}
	272	if (!defined($cur_fs) && $is_default) \|\| (defined($cur_fs) && $fs eq $cur_fs);
	273	}
	274
	275	return $res;
	276	};
	277
	278	__PACKAGE__->register_method ({
	279	name => 'destroyfs',
	280	path => 'destroyfs',
	281	method => 'DELETE',
	282	description => "Destroy a Ceph filesystem",
	283	parameters => {
	284	additionalProperties => 0,
	285	properties => {
	286	node => get_standard_option('pve-node'),
	287	name => {
	288	description => "The ceph filesystem name.",
	289	type => 'string',
	290	},
	291	'remove-storages' => {
	292	description => "Remove all pveceph-managed storages configured for this fs.",
	293	type => 'boolean',
	294	optional => 1,
	295	default => 0,
	296	},
	297	'remove-pools' => {
	298	description => "Remove data and metadata pools configured for this fs.",
	299	type => 'boolean',
	300	optional => 1,
	301	default => 0,
	302	},
	303	},
	304	},
	305	returns => { type => 'string' },
	306	code => sub {
	307	my ($param) = @_;
	308
	309	PVE::Ceph::Tools::check_ceph_inited();
	310
	311	my $rpcenv = PVE::RPCEnvironment::get();
	312	my $user = $rpcenv->get_user();
	313
	314	my $fs_name = $param->{name};
	315
	316	my $fs;
	317	my $fs_list = PVE::Ceph::Tools::ls_fs();
	318	for my $entry (@$fs_list) {
	319	next if $entry->{name} ne $fs_name;
	320	$fs = $entry;
	321	last;
	322	}
	323	die "no such cephfs '$fs_name'\n" if !$fs;
324
325	my $worker = sub {
326	my $rados = PVE::RADOS->new();
327
328	if ($param->{'remove-storages'}) {
329	my $defaultfs;
330	my $fs_dump = $rados->mon_command({ prefix => "fs dump" });
331	for my $fs ($fs_dump->{filesystems}->@*) {
332	next if $fs->{id} != $fs_dump->{default_fscid};
333	$defaultfs = $fs->{mdsmap}->{fs_name};
334	}
335	warn "no default fs found, maybe not all relevant storages are removed\n"
336	if !defined($defaultfs);
337
338	my $storages = $get_storages->($fs_name, $fs_name eq ($defaultfs // ''));
339	for my $storeid (keys %$storages) {
340	my $store = $storages->{$storeid};
341	if (!$store->{disable}) {
342	die "storage '$storeid' is not disabled, make sure to disable ".
343	"and unmount the storage first\n";
344	}
345	}
346
347	my $err;
348	for my $storeid (keys %$storages) {
349	# skip external clusters, not managed by pveceph
350	next if $storages->{$storeid}->{monhost};
351	eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
352	if ($@) {
353	warn "failed to remove storage '$storeid': $@\n";
354	$err = 1;
355	}
356	}
357	die "failed to remove (some) storages - check log and remove manually!\n"
358	if $err;
359	}
360
361	PVE::Ceph::Tools::destroy_fs($fs_name, $rados);
362
363	if ($param->{'remove-pools'}) {
364	warn "removing metadata pool '$fs->{metadata_pool}'\n";
365	eval { PVE::Ceph::Tools::destroy_pool($fs->{metadata_pool}, $rados) };
366	warn "$@\n" if $@;
367
368	foreach my $pool ($fs->{data_pools}->@*) {
369	warn "removing data pool '$pool'\n";
370	eval { PVE::Ceph::Tools::destroy_pool($pool, $rados) };
371	warn "$@\n" if $@;
372	}
373	}
374
375	};
376	return $rpcenv->fork_worker('cephdestroyfs', $fs_name, $user, $worker);
377	}});
378
b48ca5a7	379	__PACKAGE__->register_method ({
cf14758f TL	380	name => 'osd-details',
cf14758f TL	381	path => 'osd-details',
b48ca5a7 AL	382	method => 'GET',
	383	description => "Get OSD details.",
	384	parameters => {
	385	additionalProperties => 0,
	386	properties => {
	387	node => get_standard_option('pve-node'),
	388	osdid => {
	389	description => "ID of the OSD",
	390	type => 'string',
	391	},
	392	verbose => {
	393	description => "Print verbose information, same as json-pretty output format.",
	394	type => 'boolean',
	395	default => 0,
	396	optional => 1,
	397	},
	398	},
	399	},
	400	returns => { type => 'object' },
	401	code => sub {
	402	my ($param) = @_;
cf14758f	403
b48ca5a7	404	PVE::Ceph::Tools::check_ceph_inited();
cf14758f	405
b48ca5a7	406	my $res = PVE::API2::Ceph::OSD->osddetails({
cf14758f TL	407	osdid => $param->{osdid},
	408	node => $param->{node},
	409	});
	410
	411	for my $dev ($res->{devices}->@*) {
	412	$dev->{"lv-info"} = PVE::API2::Ceph::OSD->osdvolume({
b48ca5a7 AL	413	osdid => $param->{osdid},
b48ca5a7 AL	414	node => $param->{node},
cf14758f	415	type => $dev->{device},
b48ca5a7	416	});
b48ca5a7 AL	417	}
	418	$res->{verbose} = 1 if $param->{verbose};
	419	return $res;
	420	}});
	421
	422	my $format_osddetails = sub {
	423	my ($data, $schema, $options) = @_;
cf14758f	424
b48ca5a7 AL	425	$options->{"output-format"} //= "text";
	426
	427	if ($data->{verbose}) {
	428	$options->{"output-format"} = "json-pretty";
	429	delete $data->{verbose};
	430	}
	431
	432	if ($options->{"output-format"} eq "text") {
cf14758f TL	433	for my $dev ($data->{devices}->@*) {
	434	my ($disk, $type, $device) = $dev->@{'physical_device', 'type', 'device'};
	435	my ($lv_size, $lv_ctime) = $dev->{'lv-info'}->@{'lv_size', 'creation_time'};
b48ca5a7	436
cf14758f	437	$data->{osd}->{$device} = "Disk: $disk, Type: $type, LV Size: $lv_size, LV Creation Time: $lv_ctime";
b48ca5a7 AL	438	}
	439	PVE::CLIFormatter::print_api_result($data->{osd}, $schema, undef, $options);
	440	} else {
	441	PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
	442	}
	443	};
	444
3e8560ac DM	445	our $cmddef = {
3e8560ac DM	446	init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ],
8c476782	447	pool => {
a0665001	448	ls => [ 'PVE::API2::Ceph::Pool', 'lspools', [], { node => $nodename }, sub {
d4dba076 AA	449	my ($data, $schema, $options) = @_;
	450	PVE::CLIFormatter::print_api_result($data, $schema,
	451	[
	452	'pool_name',
	453	'size',
	454	'min_size',
	455	'pg_num',
5a3d7942 AA	456	'pg_num_min',
5a3d7942 AA	457	'pg_num_final',
d4dba076	458	'pg_autoscale_mode',
5a3d7942 AA	459	'target_size',
5a3d7942 AA	460	'target_size_ratio',
d4dba076 AA	461	'crush_rule_name',
	462	'percent_used',
	463	'bytes_used',
	464	],
	465	$options);
	466	}, $PVE::RESTHandler::standard_output_options],
a0665001 AL	467	create => [ 'PVE::API2::Ceph::Pool', 'createpool', ['name'], { node => $nodename }],
	468	destroy => [ 'PVE::API2::Ceph::Pool', 'destroypool', ['name'], { node => $nodename } ],
	469	set => [ 'PVE::API2::Ceph::Pool', 'setpool', ['name'], { node => $nodename } ],
	470	get => [ 'PVE::API2::Ceph::Pool', 'getpool', ['name'], { node => $nodename }, sub {
54ba7dd9 AA	471	my ($data, $schema, $options) = @_;
	472	PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
	473	}, $PVE::RESTHandler::standard_output_options],
8c476782 TL	474	},
	475	lspools => { alias => 'pool ls' },
	476	createpool => { alias => 'pool create' },
	477	destroypool => { alias => 'pool destroy' },
	478	fs => {
	479	create => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
02c1e98e	480	destroy => [ __PACKAGE__, 'destroyfs', ['name'], { node => $nodename }],
8c476782 TL	481	},
8c476782 TL	482	osd => {
79fa41a2 DC	483	create => [ 'PVE::API2::Ceph::OSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
79fa41a2 DC	484	destroy => [ 'PVE::API2::Ceph::OSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
cf14758f TL	485	details => [
	486	__PACKAGE__, 'osd-details', ['osdid'], { node => $nodename }, $format_osddetails,
	487	$PVE::RESTHandler::standard_output_options,
	488	],
8c476782 TL	489	},
	490	createosd => { alias => 'osd create' },
	491	destroyosd => { alias => 'osd destroy' },
	492	mon => {
98fe93ae DC	493	create => [ 'PVE::API2::Ceph::MON', 'createmon', [], { node => $nodename }, $upid_exit],
98fe93ae DC	494	destroy => [ 'PVE::API2::Ceph::MON', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
8c476782 TL	495	},
	496	createmon => { alias => 'mon create' },
	497	destroymon => { alias => 'mon destroy' },
	498	mgr => {
4fec2764 DC	499	create => [ 'PVE::API2::Ceph::MGR', 'createmgr', [], { node => $nodename }, $upid_exit],
4fec2764 DC	500	destroy => [ 'PVE::API2::Ceph::MGR', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
8c476782 TL	501	},
	502	createmgr => { alias => 'mgr create' },
	503	destroymgr => { alias => 'mgr destroy' },
	504	mds => {
	505	create => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
f16bb531	506	destroy => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['name'], { node => $nodename }, $upid_exit],
8c476782	507	},
7da6ff26 DJ	508	start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename }, $upid_exit],
7da6ff26 DJ	509	stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename }, $upid_exit],
3e8560ac DM	510	install => [ __PACKAGE__, 'install', [] ],
3e8560ac DM	511	purge => [ __PACKAGE__, 'purge', [] ],
1baeb2d1	512	status => [ __PACKAGE__, 'status', []],
3e8560ac DM	513	};
	514
	515	1;