[pve-manager-legacy.git] / PVE / API2 / Ceph / MON.pm

package PVE::API2::Ceph::MON;

use strict;
use warnings;

use Net::IP;
use File::Path;

use PVE::Ceph::Tools;
use PVE::Ceph::Services;
use PVE::Cluster qw(cfs_read_file cfs_write_file);
use PVE::JSONSchema qw(get_standard_option);
use PVE::Network;
use PVE::RADOS;
use PVE::RESTHandler;
use PVE::RPCEnvironment;
use PVE::Tools qw(run_command file_set_contents);
use PVE::CephConfig;
use PVE::API2::Ceph::MGR;

use base qw(PVE::RESTHandler);

my $find_mon_ips = sub {
    my ($cfg, $rados, $node, $mon_address) = @_;

    my $overwrite_ips = [ PVE::Tools::split_list($mon_address) ];
    $overwrite_ips = PVE::Network::unique_ips($overwrite_ips);

    my $pubnet;
    if ($rados) {
	$pubnet = $rados->mon_command({ prefix => "config get" , who => "mon.",
		key => "public_network", format => 'plain' });
	# if not defined in the db, the result is empty, it is also always
	# followed by a newline
	($pubnet) = $pubnet =~ m/^(\S+)$/;
    }
    $pubnet //= $cfg->{global}->{public_network};

    if (!$pubnet) {
	if (scalar(@{$overwrite_ips})) {
	    return $overwrite_ips;
	} else {
	   # don't refactor into '[ PVE::Cluster::remote... ]' as it uses wantarray
	   my $ip = PVE::Cluster::remote_node_ip($node);
	   return [ $ip ];
	}
    }

    my $public_nets = [ PVE::Tools::split_list($pubnet) ];
    if (scalar(@{$public_nets}) > 1) {
	warn "Multiple Ceph public networks detected on $node: $pubnet\n";
	warn "Networks must be capable of routing to each other.\n";
    }

    my $res = [];

    if (!scalar(@{$overwrite_ips})) { # auto-select one address for each public network
	for my $net (@{$public_nets}) {
	    my $allowed_ips = PVE::Network::get_local_ip_from_cidr($net);
	    $allowed_ips = PVE::Network::unique_ips($allowed_ips);

	    die "No active IP found for the requested ceph public network '$net' on node '$node'\n"
		if scalar(@$allowed_ips) < 1;

	    if (scalar(@$allowed_ips) == 1) {
		push @{$res}, $allowed_ips->[0];
	    } else {
		die "Multiple IPs for ceph public network '$net' detected on $node:\n".
		    join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
	    }
	}
    } else { # check if overwrite IPs are active and in any of the public networks
	my $allowed_list = [];

	for my $net (@{$public_nets}) {
	    push @{$allowed_list}, @{PVE::Network::get_local_ip_from_cidr($net)};
	}

	my $allowed_ips = PVE::Network::unique_ips($allowed_list);

	for my $overwrite_ip (@{$overwrite_ips}) {
	    die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n"
		if !grep { $_ eq $overwrite_ip } @{$allowed_ips};

	    push @{$res}, $overwrite_ip;
	}
    }

    return $res;
};

my $ips_from_mon_host = sub {
    my ($mon_host) = @_;

    my $ips = [];

    my @hosts = PVE::Tools::split_list($mon_host);

    for my $host (@hosts) {
	$host =~ s|^\[?v\d+\:||; # remove beginning of vector
	$host =~ s|/\d+\]?||; # remove end of vector

	($host) = PVE::Tools::parse_host_and_port($host);
	next if !defined($host);

	# filter out hostnames
	my $ip = PVE::JSONSchema::pve_verify_ip($host, 1);
	next if !defined($ip);

	push @{$ips}, $ip;
    }

    return $ips;
};

my $assert_mon_prerequisites = sub {
    my ($cfg, $monhash, $monid, $monips) = @_;

    my $used_ips = {};

    my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});

    for my $mon_host_ip (@{$mon_host_ips}) {
	my $ip = PVE::Network::canonical_ip($mon_host_ip);
	$used_ips->{$ip} = 1;
    }

    for my $mon (values %{$monhash}) {
	next if !defined($mon->{addr});

	for my $ip ($ips_from_mon_host->($mon->{addr})->@*) {
	    $ip = PVE::Network::canonical_ip($ip);
	    $used_ips->{$ip} = 1;
	}
    }

    for my $monip (@{$monips}) {
	$monip = PVE::Network::canonical_ip($monip);
	die "monitor address '$monip' already in use\n" if $used_ips->{$monip};
    }

    if (defined($monhash->{$monid})) {
	die "monitor '$monid' already exists\n";
    }
};

my $assert_mon_can_remove = sub {
    my ($monhash, $monlist, $monid, $mondir) = @_;

    if (!(defined($monhash->{"mon.$monid"}) ||
	  grep { $_->{name} && $_->{name} eq $monid } @$monlist))
    {
	die "no such monitor id '$monid'\n"
    }

    die "monitor filesystem '$mondir' does not exist on this node\n" if ! -d $mondir;
    die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
};

my $remove_addr_from_mon_host = sub {
    my ($monhost, $addr) = @_;

    $addr = "[$addr]" if PVE::JSONSchema::pve_verify_ipv6($addr, 1);

    # various replaces to remove the ip
    # we always match the beginning or a separator (also at the end)
    # so we do not accidentally remove a wrong ip
    # e.g. removing 10.0.0.1 should not remove 10.0.0.101 or 110.0.0.1

    # remove vector containing this ip
    # format is [vX:ip:port/nonce,vY:ip:port/nonce]
    my $vectorpart_re = "v\\d+:\Q$addr\E:\\d+\\/\\d+";
    $monhost =~ s/(^|[ ,;]*)\[$vectorpart_re(?:,$vectorpart_re)*\](?:[ ,;]+|$)/$1/;

    # ip (+ port)
    $monhost =~ s/(^|[ ,;]+)\Q$addr\E(?::\d+)?(?:[ ,;]+|$)/$1/;

    # ipv6 only without brackets
    if ($addr =~ m/^\[?(.*?:.*?)\]?$/) {
	$addr = $1;
	$monhost =~ s/(^|[ ,;]+)\Q$addr\E(?:[ ,;]+|$)/$1/;
    }

    # remove trailing separators
    $monhost =~ s/[ ,;]+$//;

    return $monhost;
};

__PACKAGE__->register_method ({
    name => 'listmon',
    path => '',
    method => 'GET',
    description => "Get Ceph monitor list.",
    proxyto => 'node',
    protected => 1,
    permissions => {
	check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	},
    },
    returns => {
	type => 'array',
	items => {
	    type => "object",
	    properties => {
		addr => { type => 'string', optional => 1 },
		ceph_version => { type => 'string', optional => 1 },
		ceph_version_short => { type => 'string', optional => 1 },
		direxists => { type => 'string', optional => 1 },
		host => { type => 'boolean', optional => 1 },
		name => { type => 'string' },
		quorum => { type => 'boolean', optional => 1 },
		rank => { type => 'integer', optional => 1 },
		service => { type => 'integer', optional => 1 },
		state => { type => 'string', optional => 1 },
	    },
	},
	links => [ { rel => 'child', href => "{name}" } ],
    },
    code => sub {
	my ($param) = @_;

	PVE::Ceph::Tools::check_ceph_inited();

	my $res = [];

	my $cfg = cfs_read_file('ceph.conf');

	my $rados = eval { PVE::RADOS->new() };
	warn $@ if $@;
	my $monhash = PVE::Ceph::Services::get_services_info("mon", $cfg, $rados);

	if ($rados) {
	    my $monstat = $rados->mon_command({ prefix => 'quorum_status' });

	    my $mons = $monstat->{monmap}->{mons};
	    foreach my $d (@$mons) {
		next if !defined($d->{name});
		my $name = $d->{name};
		$monhash->{$name}->{rank} = $d->{rank};
		$monhash->{$name}->{addr} = $d->{addr};
		if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
		    $monhash->{$name}->{quorum} = 1;
		    $monhash->{$name}->{state} = 'running';
		}
	    }

	} else {
	    # we cannot check the status if we do not have a RADOS
	    # object, so set the state to unknown
	    foreach my $monid (sort keys %$monhash) {
		$monhash->{$monid}->{state} = 'unknown';
	    }
	}

	return PVE::RESTHandler::hash_to_array($monhash, 'name');
    }});

__PACKAGE__->register_method ({
    name => 'createmon',
    path => '{monid}',
    method => 'POST',
    description => "Create Ceph Monitor and Manager",
    proxyto => 'node',
    protected => 1,
    permissions => {
	check => ['perm', '/', [ 'Sys.Modify' ]],
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    monid => {
		type => 'string',
		optional => 1,
		pattern => PVE::Ceph::Services::SERVICE_REGEX,
		maxLength => 200,
		description => "The ID for the monitor, when omitted the same as the nodename",
	    },
	    'mon-address' => {
		description => 'Overwrites autodetected monitor IP address(es). ' .
		               'Must be in the public network(s) of Ceph.',
		type => 'string', format => 'ip-list',
		optional => 1,
	    },
	},
    },
    returns => { type => 'string' },
    code => sub {
	my ($param) = @_;

	PVE::Ceph::Tools::check_ceph_installed('ceph_mon');
	PVE::Ceph::Tools::check_ceph_inited();
	PVE::Ceph::Tools::setup_pve_symlinks();

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $cfg = cfs_read_file('ceph.conf');
	my $rados = eval { PVE::RADOS->new() }; # try a rados connection, fails for first monitor
	my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);

	my $is_first_monitor = !(scalar(keys %$monhash) || $cfg->{global}->{mon_host});

	if (!defined($rados) && !$is_first_monitor) {
	    die "Could not connect to ceph cluster despite configured monitors\n";
	}

	my $monid = $param->{monid} // $param->{node};
	my $monsection = "mon.$monid";
	my $ips = $find_mon_ips->($cfg, $rados, $param->{node}, $param->{'mon-address'});

	$assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);

	my $worker = sub  {
	    my $upid = shift;

	    PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
		# update cfg content and reassert prereqs inside the lock
		$cfg = cfs_read_file('ceph.conf');
		# reopen with longer timeout
		if (defined($rados)) {
		    $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
		}
		$monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
		$assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);

		my $client_keyring = PVE::Ceph::Tools::get_or_create_admin_keyring();
		my $mon_keyring = PVE::Ceph::Tools::get_config('pve_mon_key_path');

		if (! -f $mon_keyring) {
		    print "creating new monitor keyring\n";
		    run_command([
			'ceph-authtool',
			'--create-keyring',
			$mon_keyring,
			'--gen-key',
			'-n',
			'mon.',
			'--cap',
			'mon',
			'allow *',
		    ]);
		    run_command([
			'ceph-authtool',
			$mon_keyring,
			'--import-keyring',
			$client_keyring,
		    ]);
		}

		my $ccname = PVE::Ceph::Tools::get_config('ccname');
		my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
		-d $mondir && die "monitor filesystem '$mondir' already exist\n";

		my $monmap = "/tmp/monmap";

		eval {
		    mkdir $mondir;

		    run_command(['chown', 'ceph:ceph', $mondir]);

		    my $is_first_address = !defined($rados);

		    my $monaddrs = [];

		    for my $ip (@{$ips}) {
			if (Net::IP::ip_is_ipv6($ip)) {
			    $cfg->{global}->{ms_bind_ipv6} = 'true';
			    $cfg->{global}->{ms_bind_ipv4} = 'false' if $is_first_address;
			} else {
			    $cfg->{global}->{ms_bind_ipv4} = 'true';
			    $cfg->{global}->{ms_bind_ipv6} = 'false' if $is_first_address;
			}

			my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]" : $ip;
			push @{$monaddrs}, "v2:$monaddr:3300";
			push @{$monaddrs}, "v1:$monaddr:6789";

			$is_first_address = 0;
		    }

		    my $monmaptool_cmd = [
			'monmaptool',
			'--clobber',
			'--addv',
			$monid,
			"[" . join(',', @{$monaddrs}) . "]",
			'--print',
			$monmap,
		    ];

		    if (defined($rados)) { # we can only have a RADOS object if we have a monitor
			my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
			file_set_contents($monmap, $mapdata);
			run_command($monmaptool_cmd);
		    } else { # we need to create a monmap for the first monitor
			push @{$monmaptool_cmd}, '--create';
			run_command($monmaptool_cmd);
		    }

		    run_command([
			'ceph-mon',
			'--mkfs',
			'-i',
			$monid,
			'--monmap',
			$monmap,
			'--keyring',
			$mon_keyring,
		    ]);
		    run_command(['chown', 'ceph:ceph', '-R', $mondir]);
		};
		my $err = $@;
		unlink $monmap;
		if ($err) {
		    File::Path::remove_tree($mondir);
		    die $err;
		}

		# update ceph.conf
		my $monhost = $cfg->{global}->{mon_host} // "";
		# add all known monitor ips to mon_host if it does not exist
		if (!defined($cfg->{global}->{mon_host})) {
		    for my $mon (sort keys %$monhash) {
			$monhost .= " " . $monhash->{$mon}->{addr};
		    }
		}
		$monhost .= " " . join(' ', @{$ips});
		$cfg->{global}->{mon_host} = $monhost;
		# The IP is needed in the ceph.conf for the first boot
		$cfg->{$monsection}->{public_addr} = $ips->[0];

		cfs_write_file('ceph.conf', $cfg);

		PVE::Ceph::Services::ceph_service_cmd('start', $monsection);

		if ($is_first_monitor) {
		    print "created the first monitor, assume it's safe to disable insecure global"
			." ID reclaim for new setup\n";
		    eval {
			run_command(
			    ['ceph', 'config', 'set', 'mon', 'auth_allow_insecure_global_id_reclaim', 'false'],
			    errfunc => sub { print STDERR "$_[0]\n" },
			)
		    };
		    warn "$@" if $@;
		}

		eval { PVE::Ceph::Services::ceph_service_cmd('enable', $monsection) };
		warn "Enable ceph-mon\@${monid}.service failed, do manually: $@\n" if $@;

		PVE::Ceph::Services::broadcast_ceph_services();
	    });
	    die $@ if $@;
	    # automatically create manager after the first monitor is created
	    if ($is_first_monitor) {
		PVE::API2::Ceph::MGR->createmgr({
		    node => $param->{node},
		    id => $param->{node}
		})
	    }
	};

	return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
    }});

__PACKAGE__->register_method ({
    name => 'destroymon',
    path => '{monid}',
    method => 'DELETE',
    description => "Destroy Ceph Monitor and Manager.",
    proxyto => 'node',
    protected => 1,
    permissions => {
	check => ['perm', '/', [ 'Sys.Modify' ]],
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    monid => {
		description => 'Monitor ID',
		type => 'string',
		pattern => PVE::Ceph::Services::SERVICE_REGEX,
	    },
	},
    },
    returns => { type => 'string' },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();

	my $authuser = $rpcenv->get_user();

	PVE::Ceph::Tools::check_ceph_inited();

	my $cfg = cfs_read_file('ceph.conf');

	my $monid = $param->{monid};
	my $monsection = "mon.$monid";

	my $rados = PVE::RADOS->new();
	my $monstat = $rados->mon_command({ prefix => 'quorum_status' });
	my $monlist = $monstat->{monmap}->{mons};
	my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);

	my $ccname = PVE::Ceph::Tools::get_config('ccname');
	my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";

	$assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);

	my $worker = sub {
	    my $upid = shift;
	    PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
		# reload info and recheck
		$cfg = cfs_read_file('ceph.conf');

		# reopen with longer timeout
		$rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
		$monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
		$monstat = $rados->mon_command({ prefix => 'quorum_status' });
		$monlist = $monstat->{monmap}->{mons};

		my $addrs = [];

		my $add_addr = sub {
		    my ($addr) = @_;

		    # extract the ip without port and nonce (if present)
		    ($addr) = $addr =~ m|^(.*):\d+(/\d+)?$|;
		    ($addr) = $addr =~ m|^\[?(.*?)\]?$|; # remove brackets
		    push @{$addrs}, $addr;
		};

		for my $mon (@$monlist) {
		    if ($mon->{name} eq $monid) {
			if ($mon->{public_addrs} && $mon->{public_addrs}->{addrvec}) {
			    my $addrvec = $mon->{public_addrs}->{addrvec};
			    for my $addr (@{$addrvec}) {
				$add_addr->($addr->{addr});
			    }
			} else {
			    $add_addr->($mon->{public_addr} // $mon->{addr});
			}
			last;
		    }
		}

		$assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);

		# this also stops the service
		$rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });

		# delete section
		delete $cfg->{$monsection};

		# delete from mon_host
		if (my $monhost = $cfg->{global}->{mon_host}) {
		    my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});

		    for my $addr (@{$addrs}) {
			$monhost = $remove_addr_from_mon_host->($monhost, $addr);

			# also remove matching IPs that differ syntactically
			if (PVE::JSONSchema::pve_verify_ip($addr, 1)) {
			    $addr = PVE::Network::canonical_ip($addr);

			    for my $mon_host_ip (@{$mon_host_ips}) {
				# match canonical addresses, but remove as present in mon_host
				if (PVE::Network::canonical_ip($mon_host_ip) eq $addr) {
				    $monhost = $remove_addr_from_mon_host->($monhost, $mon_host_ip);
				}
			    }
			}
		    }
		    $cfg->{global}->{mon_host} = $monhost;
		}

		cfs_write_file('ceph.conf', $cfg);
		File::Path::remove_tree($mondir);
		eval { PVE::Ceph::Services::ceph_service_cmd('disable', $monsection) };
		warn $@ if $@;
		PVE::Ceph::Services::broadcast_ceph_services();
	    });

	    die $@ if $@;
	};

	return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
    }});

1;
Commit	Line	Data
d09e0046 DC	1	package PVE::API2::Ceph::MON;
	2
	3	use strict;
	4	use warnings;
	5
	6	use Net::IP;
	7	use File::Path;
	8
	9	use PVE::Ceph::Tools;
	10	use PVE::Ceph::Services;
	11	use PVE::Cluster qw(cfs_read_file cfs_write_file);
	12	use PVE::JSONSchema qw(get_standard_option);
	13	use PVE::Network;
	14	use PVE::RADOS;
	15	use PVE::RESTHandler;
	16	use PVE::RPCEnvironment;
	17	use PVE::Tools qw(run_command file_set_contents);
de71fb96	18	use PVE::CephConfig;
9edf207c	19	use PVE::API2::Ceph::MGR;
d09e0046 DC	20
	21	use base qw(PVE::RESTHandler);
	22
05c5bd7c AA	23	my $find_mon_ips = sub {
	24	my ($cfg, $rados, $node, $mon_address) = @_;
	25
	26	my $overwrite_ips = [ PVE::Tools::split_list($mon_address) ];
	27	$overwrite_ips = PVE::Network::unique_ips($overwrite_ips);
1c89d71b DC	28
	29	my $pubnet;
	30	if ($rados) {
	31	$pubnet = $rados->mon_command({ prefix => "config get" , who => "mon.",
	32	key => "public_network", format => 'plain' });
	33	# if not defined in the db, the result is empty, it is also always
	34	# followed by a newline
	35	($pubnet) = $pubnet =~ m/^(\S+)$/;
	36	}
	37	$pubnet //= $cfg->{global}->{public_network};
d09e0046 DC	38
d09e0046 DC	39	if (!$pubnet) {
05c5bd7c AA	40	if (scalar(@{$overwrite_ips})) {
	41	return $overwrite_ips;
	42	} else {
	43	# don't refactor into '[ PVE::Cluster::remote... ]' as it uses wantarray
	44	my $ip = PVE::Cluster::remote_node_ip($node);
	45	return [ $ip ];
	46	}
d09e0046 DC	47	}
d09e0046 DC	48
05c5bd7c AA	49	my $public_nets = [ PVE::Tools::split_list($pubnet) ];
	50	if (scalar(@{$public_nets}) > 1) {
	51	warn "Multiple Ceph public networks detected on $node: $pubnet\n";
	52	warn "Networks must be capable of routing to each other.\n";
	53	}
	54
	55	my $res = [];
	56
	57	if (!scalar(@{$overwrite_ips})) { # auto-select one address for each public network
	58	for my $net (@{$public_nets}) {
05c5bd7c AA	59	my $allowed_ips = PVE::Network::get_local_ip_from_cidr($net);
05c5bd7c AA	60	$allowed_ips = PVE::Network::unique_ips($allowed_ips);
56a64429	61
05c5bd7c AA	62	die "No active IP found for the requested ceph public network '$net' on node '$node'\n"
05c5bd7c AA	63	if scalar(@$allowed_ips) < 1;
d09e0046	64
05c5bd7c AA	65	if (scalar(@$allowed_ips) == 1) {
	66	push @{$res}, $allowed_ips->[0];
	67	} else {
	68	die "Multiple IPs for ceph public network '$net' detected on $node:\n".
	69	join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
	70	}
d09e0046	71	}
05c5bd7c AA	72	} else { # check if overwrite IPs are active and in any of the public networks
	73	my $allowed_list = [];
	74
	75	for my $net (@{$public_nets}) {
05c5bd7c	76	push @{$allowed_list}, @{PVE::Network::get_local_ip_from_cidr($net)};
d09e0046	77	}
d09e0046	78
05c5bd7c AA	79	my $allowed_ips = PVE::Network::unique_ips($allowed_list);
	80
	81	for my $overwrite_ip (@{$overwrite_ips}) {
	82	die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n"
	83	if !grep { $_ eq $overwrite_ip } @{$allowed_ips};
	84
	85	push @{$res}, $overwrite_ip;
	86	}
d09e0046	87	}
05c5bd7c AA	88
05c5bd7c AA	89	return $res;
d09e0046 DC	90	};
d09e0046 DC	91
5b4053f2 FE	92	my $ips_from_mon_host = sub {
	93	my ($mon_host) = @_;
	94
	95	my $ips = [];
	96
	97	my @hosts = PVE::Tools::split_list($mon_host);
	98
	99	for my $host (@hosts) {
	100	$host =~ s\|^\[?v\d+\:\|\|; # remove beginning of vector
	101	$host =~ s\|/\d+\]?\|\|; # remove end of vector
	102
	103	($host) = PVE::Tools::parse_host_and_port($host);
	104	next if !defined($host);
	105
	106	# filter out hostnames
	107	my $ip = PVE::JSONSchema::pve_verify_ip($host, 1);
	108	next if !defined($ip);
	109
	110	push @{$ips}, $ip;
	111	}
	112
	113	return $ips;
	114	};
	115
6b617f18	116	my $assert_mon_prerequisites = sub {
05c5bd7c	117	my ($cfg, $monhash, $monid, $monips) = @_;
6b617f18	118
86ed64f9 FE	119	my $used_ips = {};
	120
	121	my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});
	122
	123	for my $mon_host_ip (@{$mon_host_ips}) {
	124	my $ip = PVE::Network::canonical_ip($mon_host_ip);
	125	$used_ips->{$ip} = 1;
	126	}
	127
	128	for my $mon (values %{$monhash}) {
	129	next if !defined($mon->{addr});
	130
61ad3f4d FE	131	for my $ip ($ips_from_mon_host->($mon->{addr})->@*) {
	132	$ip = PVE::Network::canonical_ip($ip);
	133	$used_ips->{$ip} = 1;
	134	}
6b617f18 DC	135	}
6b617f18 DC	136
05c5bd7c AA	137	for my $monip (@{$monips}) {
	138	$monip = PVE::Network::canonical_ip($monip);
	139	die "monitor address '$monip' already in use\n" if $used_ips->{$monip};
	140	}
86ed64f9	141
6b617f18 DC	142	if (defined($monhash->{$monid})) {
	143	die "monitor '$monid' already exists\n";
	144	}
	145	};
	146
64abb19b DC	147	my $assert_mon_can_remove = sub {
	148	my ($monhash, $monlist, $monid, $mondir) = @_;
	149
	150	if (!(defined($monhash->{"mon.$monid"}) \|\|
	151	grep { $_->{name} && $_->{name} eq $monid } @$monlist))
	152	{
	153	die "no such monitor id '$monid'\n"
	154	}
	155
	156	die "monitor filesystem '$mondir' does not exist on this node\n" if ! -d $mondir;
	157	die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
	158	};
	159
b5962f83 FE	160	my $remove_addr_from_mon_host = sub {
	161	my ($monhost, $addr) = @_;
	162
343975fd FE	163	$addr = "[$addr]" if PVE::JSONSchema::pve_verify_ipv6($addr, 1);
343975fd FE	164
b5962f83 FE	165	# various replaces to remove the ip
	166	# we always match the beginning or a separator (also at the end)
	167	# so we do not accidentally remove a wrong ip
	168	# e.g. removing 10.0.0.1 should not remove 10.0.0.101 or 110.0.0.1
	169
	170	# remove vector containing this ip
	171	# format is [vX:ip:port/nonce,vY:ip:port/nonce]
	172	my $vectorpart_re = "v\\d+:\Q$addr\E:\\d+\\/\\d+";
	173	$monhost =~ s/(^\|[ ,;])\[$vectorpart_re(?:,$vectorpart_re)\](?:[ ,;]+\|$)/$1/;
	174
	175	# ip (+ port)
	176	$monhost =~ s/(^\|[ ,;]+)\Q$addr\E(?::\d+)?(?:[ ,;]+\|$)/$1/;
	177
	178	# ipv6 only without brackets
	179	if ($addr =~ m/^\[?(.?:.?)\]?$/) {
	180	$addr = $1;
	181	$monhost =~ s/(^\|[ ,;]+)\Q$addr\E(?:[ ,;]+\|$)/$1/;
	182	}
	183
	184	# remove trailing separators
	185	$monhost =~ s/[ ,;]+$//;
	186
	187	return $monhost;
	188	};
	189
d09e0046 DC	190	__PACKAGE__->register_method ({
	191	name => 'listmon',
	192	path => '',
	193	method => 'GET',
	194	description => "Get Ceph monitor list.",
	195	proxyto => 'node',
	196	protected => 1,
	197	permissions => {
	198	check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
	199	},
	200	parameters => {
	201	additionalProperties => 0,
	202	properties => {
	203	node => get_standard_option('pve-node'),
	204	},
	205	},
	206	returns => {
	207	type => 'array',
	208	items => {
	209	type => "object",
	210	properties => {
4d8b2fe2	211	addr => { type => 'string', optional => 1 },
0a461549 AL	212	ceph_version => { type => 'string', optional => 1 },
	213	ceph_version_short => { type => 'string', optional => 1 },
	214	direxists => { type => 'string', optional => 1 },
	215	host => { type => 'boolean', optional => 1 },
	216	name => { type => 'string' },
	217	quorum => { type => 'boolean', optional => 1 },
	218	rank => { type => 'integer', optional => 1 },
	219	service => { type => 'integer', optional => 1 },
	220	state => { type => 'string', optional => 1 },
d09e0046 DC	221	},
	222	},
	223	links => [ { rel => 'child', href => "{name}" } ],
	224	},
	225	code => sub {
	226	my ($param) = @_;
	227
	228	PVE::Ceph::Tools::check_ceph_inited();
	229
	230	my $res = [];
	231
	232	my $cfg = cfs_read_file('ceph.conf');
	233
ae5db0aa DC	234	my $rados = eval { PVE::RADOS->new() };
	235	warn $@ if $@;
	236	my $monhash = PVE::Ceph::Services::get_services_info("mon", $cfg, $rados);
d09e0046	237
ae5db0aa	238	if ($rados) {
90396336	239	my $monstat = $rados->mon_command({ prefix => 'quorum_status' });
4d8b2fe2	240
d09e0046 DC	241	my $mons = $monstat->{monmap}->{mons};
	242	foreach my $d (@$mons) {
	243	next if !defined($d->{name});
3ddff8d1 TL	244	my $name = $d->{name};
	245	$monhash->{$name}->{rank} = $d->{rank};
	246	$monhash->{$name}->{addr} = $d->{addr};
d09e0046	247	if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
3ddff8d1 TL	248	$monhash->{$name}->{quorum} = 1;
3ddff8d1 TL	249	$monhash->{$name}->{state} = 'running';
d09e0046 DC	250	}
d09e0046 DC	251	}
4d8b2fe2	252
ae5db0aa DC	253	} else {
	254	# we cannot check the status if we do not have a RADOS
	255	# object, so set the state to unknown
	256	foreach my $monid (sort keys %$monhash) {
	257	$monhash->{$monid}->{state} = 'unknown';
	258	}
	259	}
d09e0046 DC	260
	261	return PVE::RESTHandler::hash_to_array($monhash, 'name');
	262	}});
	263
	264	__PACKAGE__->register_method ({
	265	name => 'createmon',
84a84f40	266	path => '{monid}',
d09e0046 DC	267	method => 'POST',
	268	description => "Create Ceph Monitor and Manager",
	269	proxyto => 'node',
	270	protected => 1,
	271	permissions => {
	272	check => ['perm', '/', [ 'Sys.Modify' ]],
	273	},
	274	parameters => {
	275	additionalProperties => 0,
	276	properties => {
	277	node => get_standard_option('pve-node'),
84a84f40	278	monid => {
d09e0046 DC	279	type => 'string',
d09e0046 DC	280	optional => 1,
2b12a5ac DC	281	pattern => PVE::Ceph::Services::SERVICE_REGEX,
2b12a5ac DC	282	maxLength => 200,
d09e0046 DC	283	description => "The ID for the monitor, when omitted the same as the nodename",
d09e0046 DC	284	},
d09e0046	285	'mon-address' => {
05c5bd7c AA	286	description => 'Overwrites autodetected monitor IP address(es). ' .
	287	'Must be in the public network(s) of Ceph.',
	288	type => 'string', format => 'ip-list',
d09e0046 DC	289	optional => 1,
	290	},
	291	},
	292	},
	293	returns => { type => 'string' },
	294	code => sub {
	295	my ($param) = @_;
	296
	297	PVE::Ceph::Tools::check_ceph_installed('ceph_mon');
d09e0046	298	PVE::Ceph::Tools::check_ceph_inited();
d09e0046 DC	299	PVE::Ceph::Tools::setup_pve_symlinks();
	300
	301	my $rpcenv = PVE::RPCEnvironment::get();
d09e0046 DC	302	my $authuser = $rpcenv->get_user();
	303
	304	my $cfg = cfs_read_file('ceph.conf');
1c89d71b	305	my $rados = eval { PVE::RADOS->new() }; # try a rados connection, fails for first monitor
6b617f18	306	my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
d09e0046	307
a0f82730 TL	308	my $is_first_monitor = !(scalar(keys %$monhash) \|\| $cfg->{global}->{mon_host});
	309
	310	if (!defined($rados) && !$is_first_monitor) {
6b617f18	311	die "Could not connect to ceph cluster despite configured monitors\n";
d09e0046 DC	312	}
d09e0046 DC	313
84a84f40	314	my $monid = $param->{monid} // $param->{node};
d09e0046	315	my $monsection = "mon.$monid";
05c5bd7c	316	my $ips = $find_mon_ips->($cfg, $rados, $param->{node}, $param->{'mon-address'});
d09e0046	317
05c5bd7c	318	$assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);
d09e0046 DC	319
	320	my $worker = sub {
	321	my $upid = shift;
	322
4acd90d4 DC	323	PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
	324	# update cfg content and reassert prereqs inside the lock
	325	$cfg = cfs_read_file('ceph.conf');
	326	# reopen with longer timeout
	327	if (defined($rados)) {
	328	$rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
	329	}
	330	$monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
05c5bd7c	331	$assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);
d09e0046	332
4acd90d4 DC	333	my $client_keyring = PVE::Ceph::Tools::get_or_create_admin_keyring();
	334	my $mon_keyring = PVE::Ceph::Tools::get_config('pve_mon_key_path');
	335
	336	if (! -f $mon_keyring) {
a0f82730	337	print "creating new monitor keyring\n";
e91e6d0a FE	338	run_command([
	339	'ceph-authtool',
	340	'--create-keyring',
	341	$mon_keyring,
	342	'--gen-key',
	343	'-n',
	344	'mon.',
	345	'--cap',
	346	'mon',
	347	'allow *',
	348	]);
	349	run_command([
	350	'ceph-authtool',
	351	$mon_keyring,
	352	'--import-keyring',
	353	$client_keyring,
	354	]);
4acd90d4	355	}
d09e0046	356
4acd90d4 DC	357	my $ccname = PVE::Ceph::Tools::get_config('ccname');
	358	my $mondir = "/var/lib/ceph/mon/$ccname-$monid";
	359	-d $mondir && die "monitor filesystem '$mondir' already exist\n";
d09e0046	360
4acd90d4	361	my $monmap = "/tmp/monmap";
d09e0046	362
4acd90d4 DC	363	eval {
4acd90d4 DC	364	mkdir $mondir;
d09e0046	365
e91e6d0a	366	run_command(['chown', 'ceph:ceph', $mondir]);
d09e0046	367
24f1cee0	368	my $is_first_address = !defined($rados);
24f1cee0	369
05c5bd7c AA	370	my $monaddrs = [];
	371
	372	for my $ip (@{$ips}) {
	373	if (Net::IP::ip_is_ipv6($ip)) {
	374	$cfg->{global}->{ms_bind_ipv6} = 'true';
	375	$cfg->{global}->{ms_bind_ipv4} = 'false' if $is_first_address;
	376	} else {
	377	$cfg->{global}->{ms_bind_ipv4} = 'true';
	378	$cfg->{global}->{ms_bind_ipv6} = 'false' if $is_first_address;
	379	}
	380
	381	my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]" : $ip;
	382	push @{$monaddrs}, "v2:$monaddr:3300";
	383	push @{$monaddrs}, "v1:$monaddr:6789";
	384
	385	$is_first_address = 0;
	386	}
2ab9ce36 FE	387
	388	my $monmaptool_cmd = [
	389	'monmaptool',
2ab9ce36 FE	390	'--clobber',
	391	'--addv',
	392	$monid,
05c5bd7c	393	"[" . join(',', @{$monaddrs}) . "]",
2ab9ce36 FE	394	'--print',
	395	$monmap,
	396	];
	397
4acd90d4 DC	398	if (defined($rados)) { # we can only have a RADOS object if we have a monitor
	399	my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
	400	file_set_contents($monmap, $mapdata);
3455860d	401	run_command($monmaptool_cmd);
4acd90d4	402	} else { # we need to create a monmap for the first monitor
3455860d	403	push @{$monmaptool_cmd}, '--create';
2ab9ce36	404	run_command($monmaptool_cmd);
77aa3df0	405	}
d09e0046	406
e91e6d0a FE	407	run_command([
	408	'ceph-mon',
	409	'--mkfs',
	410	'-i',
	411	$monid,
	412	'--monmap',
	413	$monmap,
	414	'--keyring',
	415	$mon_keyring,
	416	]);
	417	run_command(['chown', 'ceph:ceph', '-R', $mondir]);
4acd90d4 DC	418	};
	419	my $err = $@;
	420	unlink $monmap;
	421	if ($err) {
	422	File::Path::remove_tree($mondir);
	423	die $err;
	424	}
d09e0046	425
4acd90d4 DC	426	# update ceph.conf
4acd90d4 DC	427	my $monhost = $cfg->{global}->{mon_host} // "";
a0499169 DC	428	# add all known monitor ips to mon_host if it does not exist
	429	if (!defined($cfg->{global}->{mon_host})) {
	430	for my $mon (sort keys %$monhash) {
	431	$monhost .= " " . $monhash->{$mon}->{addr};
	432	}
	433	}
05c5bd7c	434	$monhost .= " " . join(' ', @{$ips});
4acd90d4	435	$cfg->{global}->{mon_host} = $monhost;
78fcb494	436	# The IP is needed in the ceph.conf for the first boot
05c5bd7c	437	$cfg->{$monsection}->{public_addr} = $ips->[0];
d09e0046	438
4acd90d4	439	cfs_write_file('ceph.conf', $cfg);
d09e0046	440
4acd90d4	441	PVE::Ceph::Services::ceph_service_cmd('start', $monsection);
77aa3df0	442
a0f82730 TL	443	if ($is_first_monitor) {
	444	print "created the first monitor, assume it's safe to disable insecure global"
	445	." ID reclaim for new setup\n";
	446	eval {
	447	run_command(
	448	['ceph', 'config', 'set', 'mon', 'auth_allow_insecure_global_id_reclaim', 'false'],
	449	errfunc => sub { print STDERR "$_[0]\n" },
	450	)
	451	};
	452	warn "$@" if $@;
	453	}
	454
4acd90d4 DC	455	eval { PVE::Ceph::Services::ceph_service_cmd('enable', $monsection) };
4acd90d4 DC	456	warn "Enable ceph-mon\@${monid}.service failed, do manually: $@\n" if $@;
77aa3df0	457
4acd90d4 DC	458	PVE::Ceph::Services::broadcast_ceph_services();
	459	});
	460	die $@ if $@;
9edf207c	461	# automatically create manager after the first monitor is created
52dc6be8	462	if ($is_first_monitor) {
9edf207c TM	463	PVE::API2::Ceph::MGR->createmgr({
	464	node => $param->{node},
	465	id => $param->{node}
	466	})
	467	}
d09e0046 DC	468	};
	469
	470	return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
	471	}});
	472
	473	__PACKAGE__->register_method ({
	474	name => 'destroymon',
	475	path => '{monid}',
	476	method => 'DELETE',
	477	description => "Destroy Ceph Monitor and Manager.",
	478	proxyto => 'node',
	479	protected => 1,
	480	permissions => {
	481	check => ['perm', '/', [ 'Sys.Modify' ]],
	482	},
	483	parameters => {
	484	additionalProperties => 0,
	485	properties => {
	486	node => get_standard_option('pve-node'),
	487	monid => {
	488	description => 'Monitor ID',
	489	type => 'string',
2b12a5ac	490	pattern => PVE::Ceph::Services::SERVICE_REGEX,
d09e0046	491	},
d09e0046 DC	492	},
	493	},
	494	returns => { type => 'string' },
	495	code => sub {
	496	my ($param) = @_;
	497
	498	my $rpcenv = PVE::RPCEnvironment::get();
	499
	500	my $authuser = $rpcenv->get_user();
	501
	502	PVE::Ceph::Tools::check_ceph_inited();
	503
	504	my $cfg = cfs_read_file('ceph.conf');
	505
	506	my $monid = $param->{monid};
	507	my $monsection = "mon.$monid";
	508
	509	my $rados = PVE::RADOS->new();
90396336	510	my $monstat = $rados->mon_command({ prefix => 'quorum_status' });
d09e0046	511	my $monlist = $monstat->{monmap}->{mons};
64abb19b	512	my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
d09e0046 DC	513
d09e0046 DC	514	my $ccname = PVE::Ceph::Tools::get_config('ccname');
d09e0046	515	my $mondir = "/var/lib/ceph/mon/$ccname-$monid";
d09e0046	516
64abb19b	517	$assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);
d09e0046 DC	518
	519	my $worker = sub {
	520	my $upid = shift;
2d4e64df DC	521	PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
	522	# reload info and recheck
	523	$cfg = cfs_read_file('ceph.conf');
	524
	525	# reopen with longer timeout
	526	$rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
	527	$monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
90396336	528	$monstat = $rados->mon_command({ prefix => 'quorum_status' });
2d4e64df	529	$monlist = $monstat->{monmap}->{mons};
d09e0046	530
05c5bd7c AA	531	my $addrs = [];
	532
	533	my $add_addr = sub {
	534	my ($addr) = @_;
	535
	536	# extract the ip without port and nonce (if present)
	537	($addr) = $addr =~ m\|^(.*):\d+(/\d+)?$\|;
	538	($addr) = $addr =~ m\|^\[?(.*?)\]?$\|; # remove brackets
	539	push @{$addrs}, $addr;
	540	};
	541
bd625472 DC	542	for my $mon (@$monlist) {
bd625472 DC	543	if ($mon->{name} eq $monid) {
05c5bd7c AA	544	if ($mon->{public_addrs} && $mon->{public_addrs}->{addrvec}) {
	545	my $addrvec = $mon->{public_addrs}->{addrvec};
	546	for my $addr (@{$addrvec}) {
	547	$add_addr->($addr->{addr});
	548	}
	549	} else {
	550	$add_addr->($mon->{public_addr} // $mon->{addr});
	551	}
bd625472 DC	552	last;
	553	}
	554	}
	555
2d4e64df	556	$assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);
d09e0046	557
886d9e24	558	# this also stops the service
2d4e64df	559	$rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
d09e0046	560
2d4e64df DC	561	# delete section
	562	delete $cfg->{$monsection};
	563
bd625472 DC	564	# delete from mon_host
bd625472 DC	565	if (my $monhost = $cfg->{global}->{mon_host}) {
05c5bd7c	566	my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});
343975fd	567
05c5bd7c AA	568	for my $addr (@{$addrs}) {
05c5bd7c AA	569	$monhost = $remove_addr_from_mon_host->($monhost, $addr);
343975fd	570
05c5bd7c AA	571	# also remove matching IPs that differ syntactically
	572	if (PVE::JSONSchema::pve_verify_ip($addr, 1)) {
	573	$addr = PVE::Network::canonical_ip($addr);
343975fd	574
05c5bd7c AA	575	for my $mon_host_ip (@{$mon_host_ips}) {
	576	# match canonical addresses, but remove as present in mon_host
	577	if (PVE::Network::canonical_ip($mon_host_ip) eq $addr) {
	578	$monhost = $remove_addr_from_mon_host->($monhost, $mon_host_ip);
	579	}
343975fd FE	580	}
	581	}
	582	}
	583	$cfg->{global}->{mon_host} = $monhost;
bd625472 DC	584	}
bd625472 DC	585
2d4e64df DC	586	cfs_write_file('ceph.conf', $cfg);
	587	File::Path::remove_tree($mondir);
	588	eval { PVE::Ceph::Services::ceph_service_cmd('disable', $monsection) };
	589	warn $@ if $@;
	590	PVE::Ceph::Services::broadcast_ceph_services();
	591	});
86c104d8 FG	592
86c104d8 FG	593	die $@ if $@;
d09e0046 DC	594	};
	595
	596	return $rpcenv->fork_worker('cephdestroymon', $monsection, $authuser, $worker);
	597	}});
	598
	599	1;