[pve-manager.git] / PVE / CephTools.pm

package PVE::CephTools;

use strict;
use warnings;
use File::Basename;
use File::Path;
use POSIX qw (LONG_MAX);
use Cwd qw(abs_path);
use IO::Dir;

use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);

my $ccname = 'ceph'; # ceph cluster name
my $ceph_cfgdir = "/etc/ceph";
my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";

my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";

my $ceph_bin = "/usr/bin/ceph";

my $config_hash = {
    ccname => $ccname,
    pve_ceph_cfgpath => $pve_ceph_cfgpath,
    pve_mon_key_path => $pve_mon_key_path,
    pve_ckeyring_path => $pve_ckeyring_path,
    ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
    ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
    long_rados_timeout => 60,
};

sub get_config {
    my $key = shift;

    my $value = $config_hash->{$key};

    die "no such ceph config '$key'" if !$value; 

    return $value;
}

sub verify_blockdev_path {
    my ($rel_path) = @_;

    die "missing path" if !$rel_path;
    my $path = abs_path($rel_path);
    die "failed to get absolute path to $rel_path" if !$path;

    die "got unusual device path '$path'\n" if $path !~  m|^/dev/(.*)$|;

    $path = "/dev/$1"; # untaint

    die "no such block device '$path'\n"
	if ! -b $path;
    
    return $path;
};

sub purge_all_ceph_files {
    # fixme: this is very dangerous - should we really support this function?

    unlink $ceph_cfgpath;

    unlink $pve_ceph_cfgpath;
    unlink $pve_ckeyring_path;
    unlink $pve_mon_key_path;

    unlink $ceph_bootstrap_osd_keyring;
    unlink $ceph_bootstrap_mds_keyring;

    system("rm -rf /var/lib/ceph/mon/ceph-*");

    # remove osd?
}

sub check_ceph_installed {
    my ($noerr) = @_;

    if (! -x $ceph_bin) {
	die "ceph binaries not installed\n" if !$noerr;
	return undef;
    }

    return 1;
}

sub check_ceph_inited {
    my ($noerr) = @_;

    return undef if !check_ceph_installed($noerr);
    
    if (! -f $pve_ceph_cfgpath) {
	die "pveceph configuration not initialized\n" if !$noerr;
	return undef;
    }

    return 1;
}

sub check_ceph_enabled {
    my ($noerr) = @_;

    return undef if !check_ceph_inited($noerr);

    if (! -f $ceph_cfgpath) {
	die "pveceph configuration not enabled\n" if !$noerr;
	return undef;
    }

    return 1;
}

sub parse_ceph_config {
    my ($filename) = @_;

    $filename = $pve_ceph_cfgpath if !$filename;

    my $cfg = {};

    return $cfg if ! -f $filename;

    my $fh = IO::File->new($filename, "r") ||
	die "unable to open '$filename' - $!\n";

    my $section;

    while (defined(my $line = <$fh>)) {
	$line =~ s/[;#].*$//;
	$line =~ s/^\s+//;
	$line =~ s/\s+$//;
	next if !$line;

	$section = $1 if $line =~ m/^\[(\S+)\]$/;
	if (!$section) {
	    warn "no section - skip: $line\n";
	    next;
	}

	if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) {
	    $cfg->{$section}->{$1} = $2;
	}

    }

    return $cfg;
}

sub write_ceph_config {
    my ($cfg) = @_;

    my $out = '';

    my $cond_write_sec = sub {
	my $re = shift;

	foreach my $section (keys %$cfg) {
	    next if $section !~ m/^$re$/;
	    $out .= "[$section]\n";
	    foreach my $key (sort keys %{$cfg->{$section}}) {
		$out .= "\t $key = $cfg->{$section}->{$key}\n";
	    }
	    $out .= "\n";
	}
    };

    &$cond_write_sec('global');
    &$cond_write_sec('client');
    &$cond_write_sec('mds');
    &$cond_write_sec('mds\..*');
    &$cond_write_sec('mon');
    &$cond_write_sec('osd');
    &$cond_write_sec('mon\..*');
    &$cond_write_sec('osd\..*');

    PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
}

sub setup_pve_symlinks {
    # fail if we find a real file instead of a link
    if (-f $ceph_cfgpath) {
	my $lnk = readlink($ceph_cfgpath);
	die "file '$ceph_cfgpath' already exists\n"
	    if !$lnk || $lnk ne $pve_ceph_cfgpath;
    } else {
	symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
	    die "unable to create symlink '$ceph_cfgpath' - $!\n";
    }
}

sub ceph_service_cmd {
    # ceph daemons does not call 'setsid', so we do that ourself
    # (fork_worker send KILL to whole process group) 
    PVE::Tools::run_command(['setsid', 'service', 'ceph', '-c', $pve_ceph_cfgpath, @_]);
}

sub list_disks {
    my $disklist = {};
    
    my $fd = IO::File->new("/proc/mounts", "r") ||
	die "unable to open /proc/mounts - $!\n";

    my $mounted = {};

    while (defined(my $line = <$fd>)) {
	my ($dev, $path, $fstype) = split(/\s+/, $line);
	next if !($dev && $path && $fstype);
	next if $dev !~ m|^/dev/|;
	my $real_dev = abs_path($dev);
	$mounted->{$real_dev} = $path;
    }
    close($fd);

    my $dev_is_mounted = sub {
	my ($dev) = @_;
	return $mounted->{$dev};
    };

    my $dir_is_empty = sub {
	my ($dir) = @_;

	my $dh = IO::Dir->new ($dir);
	return 1 if !$dh;
	
	while (defined(my $tmp = $dh->read)) {
	    next if $tmp eq '.' || $tmp eq '..';
	    $dh->close;
	    return 0;
	}
	$dh->close;
	return 1;
    };

    my $journal_uuid = '45b0969e-9b03-4f30-b4c6-b4b80ceff106';

    my $journalhash = {};
    dir_glob_foreach('/dev/disk/by-parttypeuuid', "$journal_uuid\..+", sub {
	my ($entry) = @_;
	my $real_dev = abs_path("/dev/disk/by-parttypeuuid/$entry");
	$journalhash->{$real_dev} = 1;
    });

    dir_glob_foreach('/sys/block', '.*', sub {
	my ($dev) = @_;

	return if $dev eq '.';
	return if $dev eq '..';

	return if $dev =~ m|^ram\d+$|; # skip ram devices
	return if $dev =~ m|^loop\d+$|; # skip loop devices
	return if $dev =~ m|^md\d+$|; # skip md devices
	return if $dev =~ m|^dm-.*$|; # skip dm related things
	return if $dev =~ m|^fd\d+$|; # skip Floppy
	return if $dev =~ m|^sr\d+$|; # skip CDs

	my $devdir = "/sys/block/$dev/device";
	return if ! -d $devdir;
	
	my $size = file_read_firstline("/sys/block/$dev/size");
	return if !$size;

	$size = $size * 512;

	my $info = `udevadm info --path /sys/block/$dev --query all`;
	return if !$info;

	return if $info !~ m/^E: DEVTYPE=disk$/m;
	return if $info =~ m/^E: ID_CDROM/m;

	my $serial = 'unknown';
	if ($info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m) {
	    $serial = $1;
	}

	my $gpt = 0;
	if ($info =~ m/^E: ID_PART_TABLE_TYPE=gpt$/m) {
	    $gpt = 1;
	}

	# detect SSD (fixme - currently only works for ATA disks)
	my $rpm = 7200; # default guess
	if ($info =~ m/^E: ID_ATA_ROTATION_RATE_RPM=(\d+)$/m) {
	    $rpm = $1;
	}

	my $vendor = file_read_firstline("$devdir/vendor") || 'unknown';
	my $model = file_read_firstline("$devdir/model") || 'unknown';

	my $used;

	$used = 'LVM' if !&$dir_is_empty("/sys/block/$dev/holders");

	$used = 'mounted' if &$dev_is_mounted("/dev/$dev");

	$disklist->{$dev} = { 
	    vendor => $vendor, 
	    model => $model, 
	    size => $size,
	    serial => $serial,
	    gpt => $gpt,
	    rmp => $rpm,
	}; 

	my $osdid = -1;

	my $journal_count = 0;

	my $found_partitions;
	my $found_lvm;
	my $found_mountpoints;
	dir_glob_foreach("/sys/block/$dev", "$dev.+", sub {
	    my ($part) = @_;

	    $found_partitions = 1;

	    if (my $mp = &$dev_is_mounted("/dev/$part")) {
		$found_mountpoints = 1;
		if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) {
		    $osdid = $1;
		} 
	    }
	    if (!&$dir_is_empty("/sys/block/$dev/$part/holders"))  {
		$found_lvm = 1;
	    }
	    $journal_count++ if $journalhash->{"/dev/$part"};
	});

	$used = 'mounted' if $found_mountpoints && !$used;
	$used = 'LVM' if $found_lvm && !$used;
	$used = 'partitions' if $found_partitions && !$used;

	$disklist->{$dev}->{used} = $used if $used;
	$disklist->{$dev}->{osdid} = $osdid;
	$disklist->{$dev}->{journals} = $journal_count;
    });

    return $disklist;
}

1;
Commit	Line	Data
a34866f0 DM	1	package PVE::CephTools;
	2
	3	use strict;
	4	use warnings;
	5	use File::Basename;
	6	use File::Path;
	7	use POSIX qw (LONG_MAX);
	8	use Cwd qw(abs_path);
7d4fc5ef	9	use IO::Dir;
a34866f0	10
7d4fc5ef	11	use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
a34866f0 DM	12
	13	my $ccname = 'ceph'; # ceph cluster name
	14	my $ceph_cfgdir = "/etc/ceph";
	15	my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
	16	my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
	17
	18	my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
	19	my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
	20	my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
	21	my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
	22
	23	my $ceph_bin = "/usr/bin/ceph";
	24
	25	my $config_hash = {
	26	ccname => $ccname,
	27	pve_ceph_cfgpath => $pve_ceph_cfgpath,
	28	pve_mon_key_path => $pve_mon_key_path,
	29	pve_ckeyring_path => $pve_ckeyring_path,
	30	ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
	31	ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
7d4fc5ef	32	long_rados_timeout => 60,
a34866f0 DM	33	};
	34
	35	sub get_config {
	36	my $key = shift;
	37
	38	my $value = $config_hash->{$key};
	39
	40	die "no such ceph config '$key'" if !$value;
	41
	42	return $value;
	43	}
	44
	45	sub verify_blockdev_path {
7fabddca	46	my ($rel_path) = @_;
a34866f0	47
7fabddca WB	48	die "missing path" if !$rel_path;
	49	my $path = abs_path($rel_path);
	50	die "failed to get absolute path to $rel_path" if !$path;
a34866f0 DM	51
	52	die "got unusual device path '$path'\n" if $path !~ m\|^/dev/(.*)$\|;
	53
	54	$path = "/dev/$1"; # untaint
	55
	56	die "no such block device '$path'\n"
	57	if ! -b $path;
	58
	59	return $path;
	60	};
	61
	62	sub purge_all_ceph_files {
	63	# fixme: this is very dangerous - should we really support this function?
	64
	65	unlink $ceph_cfgpath;
	66
	67	unlink $pve_ceph_cfgpath;
	68	unlink $pve_ckeyring_path;
	69	unlink $pve_mon_key_path;
	70
	71	unlink $ceph_bootstrap_osd_keyring;
	72	unlink $ceph_bootstrap_mds_keyring;
	73
	74	system("rm -rf /var/lib/ceph/mon/ceph-*");
	75
	76	# remove osd?
	77	}
	78
	79	sub check_ceph_installed {
	80	my ($noerr) = @_;
	81
	82	if (! -x $ceph_bin) {
	83	die "ceph binaries not installed\n" if !$noerr;
	84	return undef;
	85	}
	86
	87	return 1;
	88	}
	89
	90	sub check_ceph_inited {
	91	my ($noerr) = @_;
	92
	93	return undef if !check_ceph_installed($noerr);
	94
	95	if (! -f $pve_ceph_cfgpath) {
	96	die "pveceph configuration not initialized\n" if !$noerr;
	97	return undef;
	98	}
	99
	100	return 1;
	101	}
	102
	103	sub check_ceph_enabled {
	104	my ($noerr) = @_;
	105
	106	return undef if !check_ceph_inited($noerr);
	107
	108	if (! -f $ceph_cfgpath) {
	109	die "pveceph configuration not enabled\n" if !$noerr;
	110	return undef;
	111	}
	112
	113	return 1;
	114	}
115
116	sub parse_ceph_config {
117	my ($filename) = @_;
118
119	$filename = $pve_ceph_cfgpath if !$filename;
120
121	my $cfg = {};
122
123	return $cfg if ! -f $filename;
124
125	my $fh = IO::File->new($filename, "r") \|\|
126	die "unable to open '$filename' - $!\n";
127
128	my $section;
129
130	while (defined(my $line = <$fh>)) {
131	$line =~ s/[;#].*$//;
132	$line =~ s/^\s+//;
133	$line =~ s/\s+$//;
134	next if !$line;
135
136	$section = $1 if $line =~ m/^\[(\S+)\]$/;
137	if (!$section) {
138	warn "no section - skip: $line\n";
139	next;
140	}
141
15a5cdd1	142	if ($line =~ m/^(.?\S)\s=\s(\S.)$/) {
a34866f0 DM	143	$cfg->{$section}->{$1} = $2;
	144	}
	145
	146	}
	147
	148	return $cfg;
	149	}
	150
	151	sub write_ceph_config {
	152	my ($cfg) = @_;
	153
	154	my $out = '';
	155
	156	my $cond_write_sec = sub {
	157	my $re = shift;
	158
	159	foreach my $section (keys %$cfg) {
	160	next if $section !~ m/^$re$/;
	161	$out .= "[$section]\n";
	162	foreach my $key (sort keys %{$cfg->{$section}}) {
	163	$out .= "\t $key = $cfg->{$section}->{$key}\n";
	164	}
	165	$out .= "\n";
	166	}
	167	};
	168
	169	&$cond_write_sec('global');
19924e77	170	&$cond_write_sec('client');
0fe9bdd5 DM	171	&$cond_write_sec('mds');
0fe9bdd5 DM	172	&$cond_write_sec('mds\..*');
a34866f0 DM	173	&$cond_write_sec('mon');
	174	&$cond_write_sec('osd');
	175	&$cond_write_sec('mon\..*');
	176	&$cond_write_sec('osd\..*');
	177
	178	PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
	179	}
	180
	181	sub setup_pve_symlinks {
	182	# fail if we find a real file instead of a link
	183	if (-f $ceph_cfgpath) {
	184	my $lnk = readlink($ceph_cfgpath);
	185	die "file '$ceph_cfgpath' already exists\n"
	186	if !$lnk \|\| $lnk ne $pve_ceph_cfgpath;
	187	} else {
	188	symlink($pve_ceph_cfgpath, $ceph_cfgpath) \|\|
	189	die "unable to create symlink '$ceph_cfgpath' - $!\n";
	190	}
	191	}
	192
	193	sub ceph_service_cmd {
c3959a07 DM	194	# ceph daemons does not call 'setsid', so we do that ourself
	195	# (fork_worker send KILL to whole process group)
	196	PVE::Tools::run_command(['setsid', 'service', 'ceph', '-c', $pve_ceph_cfgpath, @_]);
a34866f0 DM	197	}
a34866f0 DM	198
7d4fc5ef DM	199	sub list_disks {
	200	my $disklist = {};
	201
	202	my $fd = IO::File->new("/proc/mounts", "r") \|\|
	203	die "unable to open /proc/mounts - $!\n";
	204
	205	my $mounted = {};
	206
	207	while (defined(my $line = <$fd>)) {
	208	my ($dev, $path, $fstype) = split(/\s+/, $line);
	209	next if !($dev && $path && $fstype);
	210	next if $dev !~ m\|^/dev/\|;
	211	my $real_dev = abs_path($dev);
	212	$mounted->{$real_dev} = $path;
	213	}
	214	close($fd);
	215
	216	my $dev_is_mounted = sub {
	217	my ($dev) = @_;
	218	return $mounted->{$dev};
	219	};
	220
22096cc9	221	my $dir_is_empty = sub {
7d4fc5ef DM	222	my ($dir) = @_;
	223
	224	my $dh = IO::Dir->new ($dir);
	225	return 1 if !$dh;
	226
	227	while (defined(my $tmp = $dh->read)) {
	228	next if $tmp eq '.' \|\| $tmp eq '..';
	229	$dh->close;
	230	return 0;
	231	}
	232	$dh->close;
	233	return 1;
	234	};
	235
	236	my $journal_uuid = '45b0969e-9b03-4f30-b4c6-b4b80ceff106';
	237
	238	my $journalhash = {};
	239	dir_glob_foreach('/dev/disk/by-parttypeuuid', "$journal_uuid\..+", sub {
	240	my ($entry) = @_;
	241	my $real_dev = abs_path("/dev/disk/by-parttypeuuid/$entry");
	242	$journalhash->{$real_dev} = 1;
	243	});
	244
	245	dir_glob_foreach('/sys/block', '.*', sub {
	246	my ($dev) = @_;
	247
	248	return if $dev eq '.';
	249	return if $dev eq '..';
	250
	251	return if $dev =~ m\|^ram\d+$\|; # skip ram devices
	252	return if $dev =~ m\|^loop\d+$\|; # skip loop devices
	253	return if $dev =~ m\|^md\d+$\|; # skip md devices
	254	return if $dev =~ m\|^dm-.*$\|; # skip dm related things
	255	return if $dev =~ m\|^fd\d+$\|; # skip Floppy
	256	return if $dev =~ m\|^sr\d+$\|; # skip CDs
	257
	258	my $devdir = "/sys/block/$dev/device";
	259	return if ! -d $devdir;
	260
	261	my $size = file_read_firstline("/sys/block/$dev/size");
	262	return if !$size;
	263
	264	$size = $size * 512;
	265
	266	my $info = `udevadm info --path /sys/block/$dev --query all`;
	267	return if !$info;
	268
	269	return if $info !~ m/^E: DEVTYPE=disk$/m;
	270	return if $info =~ m/^E: ID_CDROM/m;
	271
	272	my $serial = 'unknown';
	273	if ($info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m) {
	274	$serial = $1;
	275	}
	276
	277	my $gpt = 0;
	278	if ($info =~ m/^E: ID_PART_TABLE_TYPE=gpt$/m) {
	279	$gpt = 1;
	280	}
	281
	282	# detect SSD (fixme - currently only works for ATA disks)
	283	my $rpm = 7200; # default guess
	284	if ($info =~ m/^E: ID_ATA_ROTATION_RATE_RPM=(\d+)$/m) {
	285	$rpm = $1;
286	}
287
288	my $vendor = file_read_firstline("$devdir/vendor") \|\| 'unknown';
289	my $model = file_read_firstline("$devdir/model") \|\| 'unknown';
290
291	my $used;
292
22096cc9	293	$used = 'LVM' if !&$dir_is_empty("/sys/block/$dev/holders");
7d4fc5ef DM	294
	295	$used = 'mounted' if &$dev_is_mounted("/dev/$dev");
	296
	297	$disklist->{$dev} = {
	298	vendor => $vendor,
	299	model => $model,
	300	size => $size,
	301	serial => $serial,
	302	gpt => $gpt,
	303	rmp => $rpm,
	304	};
	305
	306	my $osdid = -1;
	307
	308	my $journal_count = 0;
	309
	310	my $found_partitions;
	311	my $found_lvm;
	312	my $found_mountpoints;
	313	dir_glob_foreach("/sys/block/$dev", "$dev.+", sub {
	314	my ($part) = @_;
	315
	316	$found_partitions = 1;
	317
	318	if (my $mp = &$dev_is_mounted("/dev/$part")) {
	319	$found_mountpoints = 1;
	320	if ($mp =~ m\|^/var/lib/ceph/osd/ceph-(\d+)$\|) {
	321	$osdid = $1;
	322	}
	323	}
22096cc9	324	if (!&$dir_is_empty("/sys/block/$dev/$part/holders")) {
7d4fc5ef DM	325	$found_lvm = 1;
	326	}
	327	$journal_count++ if $journalhash->{"/dev/$part"};
	328	});
	329
	330	$used = 'mounted' if $found_mountpoints && !$used;
	331	$used = 'LVM' if $found_lvm && !$used;
	332	$used = 'partitions' if $found_partitions && !$used;
	333
	334	$disklist->{$dev}->{used} = $used if $used;
	335	$disklist->{$dev}->{osdid} = $osdid;
	336	$disklist->{$dev}->{journals} = $journal_count;
	337	});
	338
	339	return $disklist;
	340	}
	341
a34866f0	342	1;