Fix #1210: ceph: extend pveceph purge

to clean service directories as well as disable and stop Ceph services.
Addtionally provide the option to remove crash and log information.

This patch is also in addtion to #2607, as the current cleanup doesn't
allow to re-configure Ceph, without manual steps during purge.

Signed-off-by: Alwin Antreich <a.antreich@proxmox.com>

diff --git a/PVE/CLI/pveceph.pm b/PVE/CLI/pveceph.pm
index 064ae5454bf80d73b980b1f5a04d9abd26f510e9..448d3ec13b83d6304d3a96daf488bb03134359c6 100755 (executable)
--- a/PVE/CLI/pveceph.pm
+++ b/PVE/CLI/pveceph.pm
@@ -18,6 +18,7 @@ use PVE::Storage;
 use PVE::Tools qw(run_command);
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::Ceph::Tools;
+use PVE::Ceph::Services;
 use PVE::API2::Ceph;
 use PVE::API2::Ceph::FS;
 use PVE::API2::Ceph::MDS;
@@ -49,25 +50,58 @@ __PACKAGE__->register_method ({
     parameters => {
        additionalProperties => 0,
        properties => {
+           logs => {
+               description => 'Additionally purge Ceph logs, /var/log/ceph.',
+               type => 'boolean',
+               optional => 1,
+           },
+           crash => {
+               description => 'Additionally purge Ceph crash logs, /var/lib/ceph/crash.',
+               type => 'boolean',
+               optional => 1,
+           },
        },
     },
     returns => { type => 'null' },
     code => sub {
        my ($param) = @_;
 
-       my $monstat;
+       my $message;
+       my $pools = [];
+       my $monstat = {};
+       my $mdsstat = {};
+       my $osdstat = [];
 
        eval {
            my $rados = PVE::RADOS->new();
-           my $monstat = $rados->mon_command({ prefix => 'mon_status' });
+           $pools = PVE::Ceph::Tools::ls_pools(undef, $rados);
+           $monstat = PVE::Ceph::Services::get_services_info('mon', undef, $rados);
+           $mdsstat = PVE::Ceph::Services::get_services_info('mds', undef, $rados);
+           $osdstat = $rados->mon_command({ prefix => 'osd metadata' });
        };
-       my $err = $@;
+       warn "Could not connect: $@" if $@;
+
+       my $osd = grep { $_->{hostname} eq $nodename } @$osdstat;
+       my $mds = grep { $mdsstat->{$_}->{host} eq $nodename } keys %$mdsstat;
+       my $mon = grep { $monstat->{$_}->{host} eq $nodename } keys %$monstat;
+
+       # no pools = no data
+       $message .= "- remove pools, this will !!DESTROY DATA!!\n" if @$pools;
+       $message .= "- remove active OSD on $nodename\n" if $osd;
+       $message .= "- remove active MDS on $nodename\n" if $mds;
+       $message .= "- remove other MONs, $nodename is not the last MON\n"
+           if scalar(keys %$monstat) > 1 && $mon;
+
+       # display all steps at once
+       die "Unable to purge Ceph!\n\nTo continue:\n$message" if $message;
 
-       die "detected running ceph services- unable to purge data\n"
-           if !$err;
+       my $services = PVE::Ceph::Services::get_local_services();
+       $services->{mon} = $monstat if $mon;
+       $services->{crash}->{$nodename} = { direxists => 1 } if $param->{crash};
+       $services->{logs}->{$nodename} = { direxists => 1 } if $param->{logs};
 
-       # fixme: this is dangerous - should we really support this function?
-       PVE::Ceph::Tools::purge_all_ceph_files();
+       PVE::Ceph::Tools::purge_all_ceph_services($services);
+       PVE::Ceph::Tools::purge_all_ceph_files($services);
 
        return undef;
     }});

diff --git a/PVE/Ceph/Tools.pm b/PVE/Ceph/Tools.pm
index e6225b78063321ed71a0aaffb3dfc26e3b202015..acec746ace39f703b813109a2f0a59cacd2b487e 100644 (file)
--- a/PVE/Ceph/Tools.pm
+++ b/PVE/Ceph/Tools.pm
@@ -11,6 +11,8 @@ use JSON;
 use PVE::Tools qw(run_command dir_glob_foreach);
 use PVE::Cluster qw(cfs_read_file);
 use PVE::RADOS;
+use PVE::Ceph::Services;
+use PVE::CephConfig;
 
 my $ccname = 'ceph'; # ceph cluster name
 my $ceph_cfgdir = "/etc/ceph";
@@ -42,6 +44,7 @@ my $config_hash = {
     ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
     ceph_mds_data_dir => $ceph_mds_data_dir,
     long_rados_timeout => 60,
+    ceph_cfgpath => $ceph_cfgpath,
 };
 
 sub get_local_version {
@@ -89,20 +92,64 @@ sub get_config {
 }
 
 sub purge_all_ceph_files {
-    # fixme: this is very dangerous - should we really support this function?
-
-    unlink $ceph_cfgpath;
-
-    unlink $pve_ceph_cfgpath;
-    unlink $pve_ckeyring_path;
-    unlink $pve_mon_key_path;
-
-    unlink $ceph_bootstrap_osd_keyring;
-    unlink $ceph_bootstrap_mds_keyring;
+    my ($services) = @_;
+    my $is_local_mon;
+    my $monlist = [ split(',', PVE::CephConfig::get_monaddr_list($pve_ceph_cfgpath)) ];
+
+    foreach my $service (keys %$services) {
+       my $type = $services->{$service};
+       next if (!%$type);
+
+       foreach my $name (keys %$type) {
+           my $dir_exists = $type->{$name}->{direxists};
+
+           $is_local_mon = grep($type->{$name}->{addr}, @$monlist)
+               if $service eq 'mon';
+
+           my $path = "/var/lib/ceph/$service";
+           $path = '/var/log/ceph' if $service eq 'logs';
+           if ($dir_exists) {
+               my $err;
+               File::Path::remove_tree($path, {
+                       keep_root => 1,
+                       error => \$err,
+                   });
+               warn "Error removing path, '$path'\n" if @$err;
+           }
+       }
+    }
 
-    system("rm -rf /var/lib/ceph/mon/ceph-*");
+    if (scalar @$monlist > 0 && !$is_local_mon) {
+       warn "Foreign MON address in ceph.conf. Keeping config & keyrings\n"
+    } else {
+       print "Removing config & keyring files\n";
+       foreach my $file (%$config_hash) {
+           unlink $file if (-e $file);
+       }
+    }
+}
 
-    # remove osd?
+sub purge_all_ceph_services {
+    my ($services) = @_;
+
+    foreach my $service (keys %$services) {
+       my $type = $services->{$service};
+       next if (!%$type);
+
+       foreach my $name (keys %$type) {
+           my $service_exists = $type->{$name}->{service};
+
+           if ($service_exists) {
+               eval {
+                   PVE::Ceph::Services::ceph_service_cmd('disable', "$service.$name");
+                   PVE::Ceph::Services::ceph_service_cmd('stop', "$service.$name");
+               };
+               my $err = $@ if $@;
+               warn "Could not disable/stop ceph-$service\@$name, error: $err\n"
+               if $err;
+           }
+       }
+    }
 }
 
 sub check_ceph_installed {