[pve-manager.git] / PVE / API2 / Ceph.pm

package PVE::API2::Ceph;

use strict;
use warnings;
use File::Basename;
use File::Path;
use POSIX qw (LONG_MAX);

use PVE::SafeSyslog;
use PVE::Tools qw(extract_param run_command);
use PVE::Exception qw(raise raise_param_exc);
use PVE::INotify;
use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
use PVE::AccessControl;
use PVE::Storage;
use PVE::RESTHandler;
use PVE::RPCEnvironment;
use PVE::JSONSchema qw(get_standard_option);
use JSON;

use base qw(PVE::RESTHandler);

use Data::Dumper; # fixme: remove

my $ccname = 'ceph'; # ceph cluster name
my $ceph_cfgdir = "/etc/ceph";
my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
my $ceph_mon_key_path = "$ceph_cfgdir/$ccname.mon.keyring";
my $pve_ckeyring_path = "/etc/pve/priv/$ccname.keyring";
my $ceph_ckeyring_path = "$ceph_cfgdir/$ccname.client.admin.keyring";

my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";

my $ceph_bin = "/usr/bin/ceph";

sub purge_all_ceph_files {
    # fixme: this is very dangerous - should we really support this function?

    unlink $ceph_cfgpath;
    unlink $ceph_mon_key_path;
    unlink $ceph_ckeyring_path;

    unlink $pve_ceph_cfgpath;
    unlink $pve_ckeyring_path;
    unlink $pve_mon_key_path;

    unlink $ceph_bootstrap_osd_keyring;
    unlink $ceph_bootstrap_mds_keyring;

    system("rm -rf /var/lib/ceph/mon/ceph-*");

    # remove osd?

}

my $check_ceph_installed = sub {
    my ($noerr) = @_;

    if (! -x $ceph_bin) {
	die "ceph binaries not installed\n" if !$noerr;
	return undef;
    }

    return 1;
};

my $check_ceph_inited = sub {
    my ($noerr) = @_;

    return undef if !&$check_ceph_installed($noerr);

    if (! -f $pve_ceph_cfgpath) {
	die "pveceph configuration not initialized\n" if !$noerr;
	return undef;
    }

    return 1;
};

my $check_ceph_enabled = sub {
    my ($noerr) = @_;

    return undef if !&$check_ceph_inited($noerr);

    if (! -f $ceph_cfgpath) {
	die "pveceph configuration not enabled\n" if !$noerr;
	return undef;
    }

    return 1;
};

my $force_symlink = sub {
    my ($old, $new) = @_;

    return if (-l $new) && (readlink($new) eq $old);
	
    unlink $new;
    symlink($old, $new) ||
	die "unable to create symlink '$new' - $!\n";
};

my $parse_ceph_config = sub {
    my ($filename) = @_;

    my $cfg = {};

    return $cfg if ! -f $filename;

    my $fh = IO::File->new($filename, "r") ||
	die "unable to open '$filename' - $!\n";

    my $section;

    while (defined(my $line = <$fh>)) {
	$line =~ s/[;#].*$//;
	$line =~ s/^\s+//;
	$line =~ s/\s+$//;
	next if !$line;

	$section = $1 if $line =~ m/^\[(\S+)\]$/;
	if (!$section) {
	    warn "no section - skip: $line\n";
	    next;
	}

	if ($line =~ m/^(.*\S)\s*=\s*(\S.*)$/) {
	    $cfg->{$section}->{$1} = $2;
	}

    }

    return $cfg;
};

my $run_ceph_cmd = sub {
    my ($cmd, %params) = @_;

    my $timeout = 3;

    my $oldalarm;
    eval {
	local $SIG{ALRM} = sub { die "timeout\n" };
	$oldalarm = alarm($timeout); 
	# Note: --connect-timeout does not work with current version
	# '--connect-timeout', $timeout,

	run_command(['ceph', '-c', $ceph_cfgpath, @$cmd], %params);
	alarm(0);
    };
    my $err = $@;

    alarm($oldalarm) if $oldalarm;

    die $err if $err;
};

my $run_ceph_cmd_text = sub {
    my ($cmd, %opts) = @_;

    my $out = '';

    my $quiet = delete $opts{quiet};

    my $parser = sub {
	my $line = shift;
	$out .= "$line\n";
    };

    my $errfunc = sub {
	my $line = shift;
	print "$line\n" if !$quiet;
    };

    &$run_ceph_cmd($cmd, outfunc => $parser, errfunc => $errfunc);

    return $out;
};

my $run_ceph_cmd_json = sub {
    my ($cmd, %opts) = @_;

    my $json = &$run_ceph_cmd_text([@$cmd, '--format', 'json'], %opts);

    return decode_json($json);
};

sub ceph_mon_status {
    my ($quiet) = @_;
 
    return &$run_ceph_cmd_json(['mon_status'], quiet => $quiet);

}

my $ceph_osd_status = sub {
    my ($quiet) = @_;

    return &$run_ceph_cmd_json(['osd', 'dump'], quiet => $quiet);
};

my $write_ceph_config = sub {
    my ($cfg) = @_;

    my $out = '';
    foreach my $section (keys %$cfg) {
	$out .= "[$section]\n";
	foreach my $key (sort keys %{$cfg->{$section}}) {
	    $out .= "\t $key = $cfg->{$section}->{$key}\n";
	}
	$out .= "\n";
    }

    PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
};

my $setup_pve_symlinks = sub {
    # fail if we find a real file instead of a link
    if (-f $ceph_cfgpath) {
	my $lnk = readlink($ceph_cfgpath);
	die "file '$ceph_cfgpath' already exists\n"
	    if !$lnk || $lnk ne $pve_ceph_cfgpath;
    }

    # now assume we are allowed to setup/overwrite content 
    &$force_symlink($pve_ceph_cfgpath, $ceph_cfgpath);
    &$force_symlink($pve_mon_key_path, $ceph_mon_key_path);
    &$force_symlink($pve_ckeyring_path, $ceph_ckeyring_path);
};

my $ceph_service_cmd = sub {
    run_command(['service', 'ceph', '-c', $ceph_cfgpath, @_]);
};

__PACKAGE__->register_method ({
    name => 'index',
    path => '',
    method => 'GET',
    description => "Directory index.",
    permissions => { user => 'all' },
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	},
    },
    returns => {
	type => 'array',
	items => {
	    type => "object",
	    properties => {},
	},
	links => [ { rel => 'child', href => "{name}" } ],
    },
    code => sub {
	my ($param) = @_;

	my $result = [
	    { name => 'init' },
	    { name => 'createmon' },
	    { name => 'destroymon' },
	    { name => 'createosd' },
	    { name => 'destroyosd' },
	    { name => 'stop' },
	    { name => 'start' },
	    { name => 'status' },
	    { name => 'crush' },
	];

	return $result;
    }});

__PACKAGE__->register_method ({
    name => 'init',
    path => 'init',
    method => 'POST',
    description => "Create initial ceph configuration.",
    proxyto => 'node',
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    size => {
		description => 'Number of replicas per object',
		type => 'integer',
		default => 2,
		optional => 1,
		minimum => 1,
		maximum => 3,
	    },
	    pg_bits => {
		description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)",
		type => 'integer',
		default => 9,
		optional => 1,
		minimum => 6,
		maximum => 14,
	    },
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	&$check_ceph_installed();

	-f $pve_ceph_cfgpath &&
	    die "configuration file '$pve_ceph_cfgpath' already exists.\n";

	my $pg_bits = $param->{pg_bits} || 9;
	my $size = $param->{size} || 2;

	my $global = {
	    'auth supported' => 'cephx',
	    'auth cluster required' => 'cephx',
	    'auth service required' => 'cephx',
	    'auth client required' => 'cephx',
	    'filestore xattr use omap' => 'true',
	    'osd journal size' => '1024',
	    'osd pool default size' => $size,
	    'osd pool default min size' => 1,
	    'osd pg bits' => $pg_bits,
	    'osd pgp bits' => $pg_bits,
	};

	# this does not work for default pools 
	#'osd pool default pg num' => $pg_num,
	#'osd pool default pgp num' => $pg_num, 

	&$write_ceph_config({global => $global});

	&$setup_pve_symlinks();

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'createmon',
    path => 'createmon',
    method => 'POST',
    description => "Create Ceph Monitor",
    proxyto => 'node',
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	&$check_ceph_inited();

	&$setup_pve_symlinks();

	if (! -f $pve_ckeyring_path) {
	    run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
			"--gen-key -n client.admin");
	}

	if (! -f $pve_mon_key_path) {
	    run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
	    run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
			"--cap mds 'allow *' " .
			"--cap osd 'allow *' " .
			"--cap mon 'allow *'");
	    run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
	    run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
	}


	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);

	my $moncount = 0;

	my $monaddrhash = {}; 

	foreach my $section (keys %$cfg) {
	    next if $section eq 'global';
	    my $d = $cfg->{$section};
	    if ($section =~ m/^mon\./) {
		$moncount++;
		if ($d->{'mon addr'}) {
		    $monaddrhash->{$d->{'mon addr'}} = $section;
		}
	    }
	}

	my $monid;
	for (my $i = 0; $i < 7; $i++) {
	    if (!$cfg->{"mon.$i"}) {
		$monid = $i;
		last;
	    }
	}
	die "unable to find usable monitor id\n" if !defined($monid);

	my $monsection = "mon.$monid";	
	my $monaddr = PVE::Cluster::remote_node_ip($param->{node}) . ":6789";
	my $monname = $param->{node};

	die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
	die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" 
	    if $monaddrhash->{$monaddr};


	my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
	-d $mondir && die "monitor filesystem '$mondir' already exist\n";
 
	my $monmap = "/tmp/monmap";
	
	eval {
	    mkdir $mondir;

	    if ($moncount > 0) {
		my $monstat = ceph_mon_status(); # online test
		&$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]);
	    } else {
		run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
	    }

	    run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
	};
	my $err = $@;
	unlink $monmap;
	if ($err) {
	    File::Path::remove_tree($mondir);
	    die $err;
	}

	$cfg->{$monsection} = {
	    'host' => $monname,
	    'mon addr' => $monaddr,
	};

	&$write_ceph_config($cfg);

	&$ceph_service_cmd('start', $monsection);

	return undef;

    }});

__PACKAGE__->register_method ({
    name => 'destroymon',
    path => 'destroymon',
    method => 'POST',
    description => "Destroy Ceph monitor.",
    proxyto => 'node',
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    monid => {
		description => 'Monitor ID',
		type => 'integer',
	    },
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	&$check_ceph_inited();

	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
       
	my $monid = $param->{monid};
	my $monsection = "mon.$monid";	

	my $monstat = ceph_mon_status();
	my $monlist = $monstat->{monmap}->{mons};

	die "no such monitor id '$monid'\n" 
	    if !defined($cfg->{$monsection});


	my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
	-d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";

	die "can't remove last monitor\n" if scalar(@$monlist) <= 1;

	&$run_ceph_cmd(['mon', 'remove', $monid]);

	eval { &$ceph_service_cmd('stop', $monsection); };
	warn $@ if $@;

	delete $cfg->{$monsection};
	&$write_ceph_config($cfg);
	File::Path::remove_tree($mondir);

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'stop',
    path => 'stop',
    method => 'POST',
    description => "Stop ceph services.",
    proxyto => 'node',
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	&$check_ceph_inited();

	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
	scalar(keys %$cfg) || die "no configuration\n";

	&$ceph_service_cmd('stop');

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'start',
    path => 'start',
    method => 'POST',
    description => "Start ceph services.",
    proxyto => 'node',
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	&$check_ceph_inited();

	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
	scalar(keys %$cfg) || die "no configuration\n";

	&$ceph_service_cmd('start');

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'status',
    path => 'status',
    method => 'GET',
    description => "Get ceph status.",
    proxyto => 'node',
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	},
    },
    returns => { type => 'object' },
    code => sub {
	my ($param) = @_;

	&$check_ceph_enabled();

	return &$run_ceph_cmd_json(['status'], quiet => 1);
    }});

__PACKAGE__->register_method ({
    name => 'createosd',
    path => 'createosd',
    method => 'POST',
    description => "Create OSD",
    proxyto => 'node',
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    dev => {
		description => "Block device name.",
		type => 'string',
	    }
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	&$check_ceph_inited();

	die "not fully configured - missing '$pve_ckeyring_path'\n" 
	    if ! -f $pve_ckeyring_path;

	&$setup_pve_symlinks();

	print "create OSD on $param->{dev}\n";

	-b  $param->{dev} || die "no such block device '$param->{dev}'\n";

	my $monstat = ceph_mon_status(1);
	die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
	my $fsid = $monstat->{monmap}->{fsid};

	if (! -f $ceph_bootstrap_osd_keyring) {
	    &$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]);
	};

	run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', 'xfs',
		     '--cluster', $ccname, '--cluster-uuid', $fsid,
		     '--', $param->{dev}]);

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'destroyosd',
    path => 'destroyosd',
    method => 'POST',
    description => "Destroy OSD",
    proxyto => 'node',
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    osdid => {
		description => 'OSD ID',
		type => 'integer',
	    },
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	&$check_ceph_inited();

	my $osdid = $param->{osdid};

	print "destroy OSD $param->{osdid}\n";

	# fixme: not sure what we should do here
 
	my $stat = &$ceph_osd_status();

	my $osdlist = $stat->{osds} || [];

	my $osdstat;
	foreach my $d (@$osdlist) {
	    if ($d->{osd} == $osdid) {
		$osdstat = $d;
		last;
	    }
	}
	die "no such OSD '$osdid'\n" if !$osdstat;

	die "osd is in use (in == 1)\n" if $osdstat->{in};
	#&$run_ceph_cmd(['osd', 'out', $osdid]);

	die "osd is still runnung (up == 1)\n" if $osdstat->{up};

	my $osdsection = "osd.$osdid";	

	eval { &$ceph_service_cmd('stop', $osdsection); };
	warn $@ if $@;

	print "Remove $osdsection from the CRUSH map\n";
	&$run_ceph_cmd(['osd', 'crush', 'remove', $osdid]);

	print "Remove the $osdsection authentication key.\n";
	&$run_ceph_cmd(['auth', 'del', $osdsection]);

	print "Remove OSD $osdsection\n";
	&$run_ceph_cmd(['osd', 'rm', $osdid]);

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'crush',
    path => 'crush',
    method => 'GET',
    description => "Get OSD crush map",
    proxyto => 'node',
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	},
    },
    returns => { type => 'string' },
    code => sub {
	my ($param) = @_;

	&$check_ceph_inited();

	my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);

	return $txt;
    }});
Commit	Line	Data
38db610a DM	1	package PVE::API2::Ceph;
	2
	3	use strict;
	4	use warnings;
	5	use File::Basename;
	6	use File::Path;
	7	use POSIX qw (LONG_MAX);
	8
	9	use PVE::SafeSyslog;
	10	use PVE::Tools qw(extract_param run_command);
	11	use PVE::Exception qw(raise raise_param_exc);
	12	use PVE::INotify;
	13	use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
	14	use PVE::AccessControl;
	15	use PVE::Storage;
	16	use PVE::RESTHandler;
	17	use PVE::RPCEnvironment;
	18	use PVE::JSONSchema qw(get_standard_option);
	19	use JSON;
	20
	21	use base qw(PVE::RESTHandler);
	22
	23	use Data::Dumper; # fixme: remove
	24
	25	my $ccname = 'ceph'; # ceph cluster name
	26	my $ceph_cfgdir = "/etc/ceph";
	27	my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
	28	my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
	29	my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
	30	my $ceph_mon_key_path = "$ceph_cfgdir/$ccname.mon.keyring";
	31	my $pve_ckeyring_path = "/etc/pve/priv/$ccname.keyring";
	32	my $ceph_ckeyring_path = "$ceph_cfgdir/$ccname.client.admin.keyring";
	33
	34	my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
	35	my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
	36
	37	my $ceph_bin = "/usr/bin/ceph";
	38
	39	sub purge_all_ceph_files {
	40	# fixme: this is very dangerous - should we really support this function?
	41
	42	unlink $ceph_cfgpath;
	43	unlink $ceph_mon_key_path;
	44	unlink $ceph_ckeyring_path;
	45
	46	unlink $pve_ceph_cfgpath;
	47	unlink $pve_ckeyring_path;
	48	unlink $pve_mon_key_path;
	49
	50	unlink $ceph_bootstrap_osd_keyring;
	51	unlink $ceph_bootstrap_mds_keyring;
	52
	53	system("rm -rf /var/lib/ceph/mon/ceph-*");
	54
	55	# remove osd?
	56
	57	}
	58
	59	my $check_ceph_installed = sub {
	60	my ($noerr) = @_;
	61
	62	if (! -x $ceph_bin) {
	63	die "ceph binaries not installed\n" if !$noerr;
	64	return undef;
65	}
66
67	return 1;
68	};
69
70	my $check_ceph_inited = sub {
71	my ($noerr) = @_;
72
73	return undef if !&$check_ceph_installed($noerr);
74
75	if (! -f $pve_ceph_cfgpath) {
76	die "pveceph configuration not initialized\n" if !$noerr;
77	return undef;
78	}
79
80	return 1;
81	};
82
9662c195 DM	83	my $check_ceph_enabled = sub {
	84	my ($noerr) = @_;
	85
	86	return undef if !&$check_ceph_inited($noerr);
	87
	88	if (! -f $ceph_cfgpath) {
	89	die "pveceph configuration not enabled\n" if !$noerr;
	90	return undef;
	91	}
	92
	93	return 1;
	94	};
	95
38db610a DM	96	my $force_symlink = sub {
	97	my ($old, $new) = @_;
	98
	99	return if (-l $new) && (readlink($new) eq $old);
	100
	101	unlink $new;
	102	symlink($old, $new) \|\|
	103	die "unable to create symlink '$new' - $!\n";
	104	};
	105
	106	my $parse_ceph_config = sub {
	107	my ($filename) = @_;
	108
	109	my $cfg = {};
	110
	111	return $cfg if ! -f $filename;
	112
	113	my $fh = IO::File->new($filename, "r") \|\|
	114	die "unable to open '$filename' - $!\n";
	115
	116	my $section;
	117
	118	while (defined(my $line = <$fh>)) {
	119	$line =~ s/[;#].*$//;
	120	$line =~ s/^\s+//;
	121	$line =~ s/\s+$//;
	122	next if !$line;
	123
	124	$section = $1 if $line =~ m/^\[(\S+)\]$/;
	125	if (!$section) {
	126	warn "no section - skip: $line\n";
	127	next;
	128	}
	129
	130	if ($line =~ m/^(.\S)\s=\s(\S.)$/) {
	131	$cfg->{$section}->{$1} = $2;
	132	}
	133
	134	}
	135
	136	return $cfg;
	137	};
	138
	139	my $run_ceph_cmd = sub {
	140	my ($cmd, %params) = @_;
	141
	142	my $timeout = 3;
	143
	144	my $oldalarm;
	145	eval {
	146	local $SIG{ALRM} = sub { die "timeout\n" };
	147	$oldalarm = alarm($timeout);
	148	# Note: --connect-timeout does not work with current version
	149	# '--connect-timeout', $timeout,
	150
	151	run_command(['ceph', '-c', $ceph_cfgpath, @$cmd], %params);
	152	alarm(0);
	153	};
	154	my $err = $@;
	155
	156	alarm($oldalarm) if $oldalarm;
	157
	158	die $err if $err;
38db610a DM	159	};
38db610a DM	160
2f692121	161	my $run_ceph_cmd_text = sub {
9662c195	162	my ($cmd, %opts) = @_;
38db610a	163
2f692121	164	my $out = '';
9662c195 DM	165
	166	my $quiet = delete $opts{quiet};
	167
38db610a DM	168	my $parser = sub {
38db610a DM	169	my $line = shift;
2f692121	170	$out .= "$line\n";
38db610a DM	171	};
	172
	173	my $errfunc = sub {
	174	my $line = shift;
	175	print "$line\n" if !$quiet;
	176	};
	177
2f692121 DM	178	&$run_ceph_cmd($cmd, outfunc => $parser, errfunc => $errfunc);
	179
	180	return $out;
	181	};
	182
	183	my $run_ceph_cmd_json = sub {
	184	my ($cmd, %opts) = @_;
38db610a	185
2f692121	186	my $json = &$run_ceph_cmd_text([@$cmd, '--format', 'json'], %opts);
38db610a	187
2f692121	188	return decode_json($json);
9662c195	189	};
38db610a	190
9662c195	191	sub ceph_mon_status {
38db610a	192	my ($quiet) = @_;
9662c195 DM	193
9662c195 DM	194	return &$run_ceph_cmd_json(['mon_status'], quiet => $quiet);
38db610a	195
9662c195	196	}
38db610a	197
9662c195 DM	198	my $ceph_osd_status = sub {
9662c195 DM	199	my ($quiet) = @_;
38db610a	200
9662c195	201	return &$run_ceph_cmd_json(['osd', 'dump'], quiet => $quiet);
38db610a DM	202	};
	203
	204	my $write_ceph_config = sub {
	205	my ($cfg) = @_;
	206
	207	my $out = '';
	208	foreach my $section (keys %$cfg) {
	209	$out .= "[$section]\n";
	210	foreach my $key (sort keys %{$cfg->{$section}}) {
	211	$out .= "\t $key = $cfg->{$section}->{$key}\n";
	212	}
	213	$out .= "\n";
	214	}
	215
	216	PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
	217	};
	218
	219	my $setup_pve_symlinks = sub {
	220	# fail if we find a real file instead of a link
	221	if (-f $ceph_cfgpath) {
	222	my $lnk = readlink($ceph_cfgpath);
	223	die "file '$ceph_cfgpath' already exists\n"
	224	if !$lnk \|\| $lnk ne $pve_ceph_cfgpath;
	225	}
	226
	227	# now assume we are allowed to setup/overwrite content
	228	&$force_symlink($pve_ceph_cfgpath, $ceph_cfgpath);
	229	&$force_symlink($pve_mon_key_path, $ceph_mon_key_path);
	230	&$force_symlink($pve_ckeyring_path, $ceph_ckeyring_path);
	231	};
	232
	233	my $ceph_service_cmd = sub {
	234	run_command(['service', 'ceph', '-c', $ceph_cfgpath, @_]);
	235	};
	236
	237	__PACKAGE__->register_method ({
	238	name => 'index',
	239	path => '',
	240	method => 'GET',
	241	description => "Directory index.",
	242	permissions => { user => 'all' },
	243	parameters => {
	244	additionalProperties => 0,
	245	properties => {
	246	node => get_standard_option('pve-node'),
	247	},
	248	},
	249	returns => {
	250	type => 'array',
	251	items => {
	252	type => "object",
	253	properties => {},
	254	},
	255	links => [ { rel => 'child', href => "{name}" } ],
	256	},
	257	code => sub {
	258	my ($param) = @_;
	259
	260	my $result = [
	261	{ name => 'init' },
	262	{ name => 'createmon' },
	263	{ name => 'destroymon' },
	264	{ name => 'createosd' },
	265	{ name => 'destroyosd' },
266	{ name => 'stop' },
267	{ name => 'start' },
268	{ name => 'status' },
2f692121	269	{ name => 'crush' },
38db610a DM	270	];
	271
	272	return $result;
	273	}});
	274
	275	__PACKAGE__->register_method ({
	276	name => 'init',
	277	path => 'init',
	278	method => 'POST',
	279	description => "Create initial ceph configuration.",
	280	proxyto => 'node',
	281	protected => 1,
	282	parameters => {
	283	additionalProperties => 0,
	284	properties => {
	285	node => get_standard_option('pve-node'),
	286	size => {
	287	description => 'Number of replicas per object',
	288	type => 'integer',
	289	default => 2,
	290	optional => 1,
	291	minimum => 1,
	292	maximum => 3,
	293	},
	294	pg_bits => {
	295	description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)",
	296	type => 'integer',
	297	default => 9,
	298	optional => 1,
	299	minimum => 6,
	300	maximum => 14,
	301	},
	302	},
	303	},
	304	returns => { type => 'null' },
	305	code => sub {
	306	my ($param) = @_;
	307
	308	&$check_ceph_installed();
	309
	310	-f $pve_ceph_cfgpath &&
	311	die "configuration file '$pve_ceph_cfgpath' already exists.\n";
	312
	313	my $pg_bits = $param->{pg_bits} \|\| 9;
	314	my $size = $param->{size} \|\| 2;
	315
	316	my $global = {
	317	'auth supported' => 'cephx',
	318	'auth cluster required' => 'cephx',
	319	'auth service required' => 'cephx',
	320	'auth client required' => 'cephx',
	321	'filestore xattr use omap' => 'true',
	322	'osd journal size' => '1024',
	323	'osd pool default size' => $size,
	324	'osd pool default min size' => 1,
	325	'osd pg bits' => $pg_bits,
	326	'osd pgp bits' => $pg_bits,
	327	};
	328
	329	# this does not work for default pools
	330	#'osd pool default pg num' => $pg_num,
	331	#'osd pool default pgp num' => $pg_num,
	332
	333	&$write_ceph_config({global => $global});
334
335	&$setup_pve_symlinks();
336
337	return undef;
338	}});
339
340	__PACKAGE__->register_method ({
341	name => 'createmon',
342	path => 'createmon',
343	method => 'POST',
344	description => "Create Ceph Monitor",
345	proxyto => 'node',
346	protected => 1,
347	parameters => {
348	additionalProperties => 0,
349	properties => {
350	node => get_standard_option('pve-node'),
351	},
352	},
353	returns => { type => 'null' },
354	code => sub {
355	my ($param) = @_;
356
357	&$check_ceph_inited();
358
359	&$setup_pve_symlinks();
360
361	if (! -f $pve_ckeyring_path) {
362	run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
363	"--gen-key -n client.admin");
364	}
365
366	if (! -f $pve_mon_key_path) {
367	run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
368	run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
369	"--cap mds 'allow *' " .
370	"--cap osd 'allow *' " .
371	"--cap mon 'allow *'");
372	run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
373	run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
374	}
375
376
377	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
378
379	my $moncount = 0;
380
381	my $monaddrhash = {};
382
383	foreach my $section (keys %$cfg) {
384	next if $section eq 'global';
385	my $d = $cfg->{$section};
386	if ($section =~ m/^mon\./) {
387	$moncount++;
388	if ($d->{'mon addr'}) {
389	$monaddrhash->{$d->{'mon addr'}} = $section;
390	}
391	}
392	}
393
394	my $monid;
395	for (my $i = 0; $i < 7; $i++) {
396	if (!$cfg->{"mon.$i"}) {
397	$monid = $i;
398	last;
399	}
400	}
401	die "unable to find usable monitor id\n" if !defined($monid);
402
403	my $monsection = "mon.$monid";
404	my $monaddr = PVE::Cluster::remote_node_ip($param->{node}) . ":6789";
405	my $monname = $param->{node};
406
407	die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
408	die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
409	if $monaddrhash->{$monaddr};
410
411
412	my $mondir = "/var/lib/ceph/mon/$ccname-$monid";
413	-d $mondir && die "monitor filesystem '$mondir' already exist\n";
414
415	my $monmap = "/tmp/monmap";
416
417	eval {
418	mkdir $mondir;
419
420	if ($moncount > 0) {
421	my $monstat = ceph_mon_status(); # online test
422	&$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]);
423	} else {
424	run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
425	}
426
427	run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
428	};
429	my $err = $@;
430	unlink $monmap;
431	if ($err) {
432	File::Path::remove_tree($mondir);
433	die $err;
434	}
435
436	$cfg->{$monsection} = {
437	'host' => $monname,
438	'mon addr' => $monaddr,
439	};
440
441	&$write_ceph_config($cfg);
442
443	&$ceph_service_cmd('start', $monsection);
444
445	return undef;
446
447	}});
448
449	__PACKAGE__->register_method ({
450	name => 'destroymon',
451	path => 'destroymon',
452	method => 'POST',
453	description => "Destroy Ceph monitor.",
454	proxyto => 'node',
455	protected => 1,
456	parameters => {
457	additionalProperties => 0,
458	properties => {
459	node => get_standard_option('pve-node'),
460	monid => {
461	description => 'Monitor ID',
462	type => 'integer',
463	},
464	},
465	},
466	returns => { type => 'null' },
467	code => sub {
468	my ($param) = @_;
469
470	&$check_ceph_inited();
471
472	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
473
474	my $monid = $param->{monid};
475	my $monsection = "mon.$monid";
476
477	my $monstat = ceph_mon_status();
478	my $monlist = $monstat->{monmap}->{mons};
479
480	die "no such monitor id '$monid'\n"
481	if !defined($cfg->{$monsection});
482
483
484	my $mondir = "/var/lib/ceph/mon/$ccname-$monid";
485	-d $mondir \|\| die "monitor filesystem '$mondir' does not exist on this node\n";
486
487	die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
488
489	&$run_ceph_cmd(['mon', 'remove', $monid]);
490
491	eval { &$ceph_service_cmd('stop', $monsection); };
492	warn $@ if $@;
493
494	delete $cfg->{$monsection};
495	&$write_ceph_config($cfg);
496	File::Path::remove_tree($mondir);
497
498	return undef;
499	}});
500
501	__PACKAGE__->register_method ({
502	name => 'stop',
503	path => 'stop',
504	method => 'POST',
505	description => "Stop ceph services.",
506	proxyto => 'node',
507	protected => 1,
508	parameters => {
509	additionalProperties => 0,
510	properties => {
511	node => get_standard_option('pve-node'),
512	},
513	},
514	returns => { type => 'null' },
515	code => sub {
516	my ($param) = @_;
517
518	&$check_ceph_inited();
519
520	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
521	scalar(keys %$cfg) \|\| die "no configuration\n";
522
523	&$ceph_service_cmd('stop');
524
525	return undef;
526	}});
527
528	__PACKAGE__->register_method ({
529	name => 'start',
530	path => 'start',
531	method => 'POST',
532	description => "Start ceph services.",
533	proxyto => 'node',
534	protected => 1,
535	parameters => {
536	additionalProperties => 0,
537	properties => {
538	node => get_standard_option('pve-node'),
539	},
540	},
541	returns => { type => 'null' },
542	code => sub {
543	my ($param) = @_;
544
545	&$check_ceph_inited();
546
547	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
548	scalar(keys %$cfg) \|\| die "no configuration\n";
549
550	&$ceph_service_cmd('start');
551
552	return undef;
553	}});
554
555	__PACKAGE__->register_method ({
556	name => 'status',
557	path => 'status',
558	method => 'GET',
559	description => "Get ceph status.",
560	proxyto => 'node',
561	protected => 1,
562	parameters => {
563	additionalProperties => 0,
564	properties => {
565	node => get_standard_option('pve-node'),
566	},
567	},
568	returns => { type => 'object' },
569	code => sub {
570	my ($param) = @_;
571
9662c195	572	&$check_ceph_enabled();
38db610a	573
9662c195	574	return &$run_ceph_cmd_json(['status'], quiet => 1);
38db610a DM	575	}});
	576
	577	__PACKAGE__->register_method ({
	578	name => 'createosd',
	579	path => 'createosd',
	580	method => 'POST',
	581	description => "Create OSD",
	582	proxyto => 'node',
	583	protected => 1,
	584	parameters => {
	585	additionalProperties => 0,
	586	properties => {
	587	node => get_standard_option('pve-node'),
	588	dev => {
	589	description => "Block device name.",
	590	type => 'string',
	591	}
	592	},
	593	},
	594	returns => { type => 'null' },
	595	code => sub {
	596	my ($param) = @_;
	597
	598	&$check_ceph_inited();
	599
	600	die "not fully configured - missing '$pve_ckeyring_path'\n"
	601	if ! -f $pve_ckeyring_path;
	602
	603	&$setup_pve_symlinks();
	604
	605	print "create OSD on $param->{dev}\n";
	606
	607	-b $param->{dev} \|\| die "no such block device '$param->{dev}'\n";
	608
	609	my $monstat = ceph_mon_status(1);
	610	die "unable to get fsid\n" if !$monstat->{monmap} \|\| !$monstat->{monmap}->{fsid};
	611	my $fsid = $monstat->{monmap}->{fsid};
	612
	613	if (! -f $ceph_bootstrap_osd_keyring) {
	614	&$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]);
	615	};
	616
	617	run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', 'xfs',
	618	'--cluster', $ccname, '--cluster-uuid', $fsid,
	619	'--', $param->{dev}]);
	620
	621	return undef;
	622	}});
	623
	624	__PACKAGE__->register_method ({
	625	name => 'destroyosd',
	626	path => 'destroyosd',
	627	method => 'POST',
	628	description => "Destroy OSD",
	629	proxyto => 'node',
	630	protected => 1,
	631	parameters => {
	632	additionalProperties => 0,
	633	properties => {
	634	node => get_standard_option('pve-node'),
	635	osdid => {
	636	description => 'OSD ID',
	637	type => 'integer',
	638	},
639	},
640	},
641	returns => { type => 'null' },
642	code => sub {
643	my ($param) = @_;
644
645	&$check_ceph_inited();
646
647	my $osdid = $param->{osdid};
648
649	print "destroy OSD $param->{osdid}\n";
650
651	# fixme: not sure what we should do here
652
653	my $stat = &$ceph_osd_status();
654
655	my $osdlist = $stat->{osds} \|\| [];
656
657	my $osdstat;
658	foreach my $d (@$osdlist) {
659	if ($d->{osd} == $osdid) {
660	$osdstat = $d;
661	last;
662	}
663	}
664	die "no such OSD '$osdid'\n" if !$osdstat;
665
666	die "osd is in use (in == 1)\n" if $osdstat->{in};
667	#&$run_ceph_cmd(['osd', 'out', $osdid]);
668
669	die "osd is still runnung (up == 1)\n" if $osdstat->{up};
670
671	my $osdsection = "osd.$osdid";
672
673	eval { &$ceph_service_cmd('stop', $osdsection); };
674	warn $@ if $@;
675
676	print "Remove $osdsection from the CRUSH map\n";
677	&$run_ceph_cmd(['osd', 'crush', 'remove', $osdid]);
678
679	print "Remove the $osdsection authentication key.\n";
680	&$run_ceph_cmd(['auth', 'del', $osdsection]);
681
682	print "Remove OSD $osdsection\n";
683	&$run_ceph_cmd(['osd', 'rm', $osdid]);
684
685	return undef;
686	}});
2f692121 DM	687
	688	__PACKAGE__->register_method ({
	689	name => 'crush',
	690	path => 'crush',
	691	method => 'GET',
	692	description => "Get OSD crush map",
	693	proxyto => 'node',
	694	protected => 1,
	695	parameters => {
	696	additionalProperties => 0,
	697	properties => {
	698	node => get_standard_option('pve-node'),
	699	},
	700	},
	701	returns => { type => 'string' },
	702	code => sub {
	703	my ($param) = @_;
	704
	705	&$check_ceph_inited();
	706
	707	my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
	708
	709	return $txt;
	710	}});
	711
	712