PVE/API2/Ceph/MON.pm

   1 package PVE::API2::Ceph::MON;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Net::IP;
   7 use File::Path;
   8
   9 use PVE::Ceph::Tools;
  10 use PVE::Ceph::Services;
  11 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::Network;
  14 use PVE::RADOS;
  15 use PVE::RESTHandler;
  16 use PVE::RPCEnvironment;
  17 use PVE::Tools qw(run_command file_set_contents);
  18
  19 use base qw(PVE::RESTHandler);
  20
  21 my $find_mon_ip = sub {
  22     my ($pubnet, $node, $overwrite_ip) = @_;
  23
  24     if (!$pubnet) {
  25         return $overwrite_ip // PVE::Cluster::remote_node_ip($node);
  26     }
  27
  28     my $allowed_ips = PVE::Network::get_local_ip_from_cidr($pubnet);
  29     die "No active IP found for the requested ceph public network '$pubnet' on node '$node'\n"
  30         if scalar(@$allowed_ips) < 1;
  31
  32     if (!$overwrite_ip) {
  33         if (scalar(@$allowed_ips) == 1) {
  34             return $allowed_ips->[0];
  35         }
  36         die "Multiple IPs for ceph public network '$pubnet' detected on $node:\n".
  37             join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
  38     } else {
  39         if (grep { $_ eq $overwrite_ip } @$allowed_ips) {
  40             return $overwrite_ip;
  41         }
  42         die "Monitor IP '$overwrite_ip' not in ceph public network '$pubnet'\n"
  43             if !PVE::Network::is_ip_in_cidr($overwrite_ip, $pubnet);
  44
  45         die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n";
  46     }
  47 };
  48
  49 __PACKAGE__->register_method ({
  50     name => 'listmon',
  51     path => '',
  52     method => 'GET',
  53     description => "Get Ceph monitor list.",
  54     proxyto => 'node',
  55     protected => 1,
  56     permissions => {
  57         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  58     },
  59     parameters => {
  60         additionalProperties => 0,
  61         properties => {
  62             node => get_standard_option('pve-node'),
  63         },
  64     },
  65     returns => {
  66         type => 'array',
  67         items => {
  68             type => "object",
  69             properties => {
  70                 name => { type => 'string' },
  71                 addr => { type => 'string', optional => 1 },
  72                 host => { type => 'string', optional => 1 },
  73             },
  74         },
  75         links => [ { rel => 'child', href => "{name}" } ],
  76     },
  77     code => sub {
  78         my ($param) = @_;
  79
  80         PVE::Ceph::Tools::check_ceph_inited();
  81
  82         my $res = [];
  83
  84         my $cfg = cfs_read_file('ceph.conf');
  85
  86         my $monhash = {};
  87
  88         eval {
  89             my $rados = PVE::RADOS->new();
  90             $monhash = PVE::Ceph::Services::get_services_info("mon", $cfg, $rados);
  91             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
  92
  93             my $mons = $monstat->{monmap}->{mons};
  94             foreach my $d (@$mons) {
  95                 next if !defined($d->{name});
  96                 $monhash->{$d->{name}}->{rank} = $d->{rank};
  97                 $monhash->{$d->{name}}->{addr} = $d->{addr};
  98                 $monhash->{$d->{name}}->{state} = 'running';
  99                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 100                     $monhash->{$d->{name}}->{quorum} = 1;
 101                 }
 102             }
 103
 104         };
 105         warn $@ if $@;
 106
 107         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 108     }});
 109
 110 __PACKAGE__->register_method ({
 111     name => 'createmon',
 112     path => '',
 113     method => 'POST',
 114     description => "Create Ceph Monitor and Manager",
 115     proxyto => 'node',
 116     protected => 1,
 117     permissions => {
 118         check => ['perm', '/', [ 'Sys.Modify' ]],
 119     },
 120     parameters => {
 121         additionalProperties => 0,
 122         properties => {
 123             node => get_standard_option('pve-node'),
 124             id => {
 125                 type => 'string',
 126                 optional => 1,
 127                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
 128                 description => "The ID for the monitor, when omitted the same as the nodename",
 129             },
 130             'exclude-manager' => {
 131                 type => 'boolean',
 132                 optional => 1,
 133                 default => 0,
 134                 description => "When set, only a monitor will be created.",
 135             },
 136             'mon-address' => {
 137                 description => 'Overwrites autodetected monitor IP address. ' .
 138                                'Must be in the public network of ceph.',
 139                 type => 'string', format => 'ip',
 140                 optional => 1,
 141             },
 142         },
 143     },
 144     returns => { type => 'string' },
 145     code => sub {
 146         my ($param) = @_;
 147
 148         PVE::Ceph::Tools::check_ceph_installed('ceph_mon');
 149
 150         PVE::Ceph::Tools::check_ceph_installed('ceph_mgr')
 151             if (!$param->{'exclude-manager'});
 152
 153         PVE::Ceph::Tools::check_ceph_inited();
 154
 155         PVE::Ceph::Tools::setup_pve_symlinks();
 156
 157         my $rpcenv = PVE::RPCEnvironment::get();
 158
 159         my $authuser = $rpcenv->get_user();
 160
 161         my $cfg = cfs_read_file('ceph.conf');
 162
 163         my $moncount = 0;
 164
 165         my $monaddrhash = {};
 166
 167         foreach my $section (keys %$cfg) {
 168             next if $section eq 'global';
 169             my $d = $cfg->{$section};
 170             if ($section =~ m/^mon\./) {
 171                 $moncount++;
 172                 if ($d->{'mon addr'}) {
 173                     $monaddrhash->{$d->{'mon addr'}} = $section;
 174                 }
 175             }
 176         }
 177
 178         my $monid = $param->{id} // $param->{node};
 179
 180         my $monsection = "mon.$monid";
 181         my $pubnet = $cfg->{global}->{'public network'};
 182         my $ip = $find_mon_ip->($pubnet, $param->{node}, $param->{'mon-address'});
 183
 184         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
 185         my $monname = $param->{node};
 186
 187         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
 188         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
 189             if $monaddrhash->{$monaddr};
 190
 191         my $worker = sub  {
 192             my $upid = shift;
 193
 194             my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 195
 196             if (! -f $pve_ckeyring_path) {
 197                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
 198                             "--gen-key -n client.admin");
 199             }
 200
 201             my $pve_mon_key_path = PVE::Ceph::Tools::get_config('pve_mon_key_path');
 202             if (! -f $pve_mon_key_path) {
 203                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
 204                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin " .
 205                             "--cap mds 'allow' " .
 206                             "--cap osd 'allow *' " .
 207                             "--cap mgr 'allow *' " .
 208                             "--cap mon 'allow *'");
 209                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring");
 210                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring");
 211                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
 212                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
 213             }
 214
 215             my $ccname = PVE::Ceph::Tools::get_config('ccname');
 216
 217             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 218             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
 219
 220             my $monmap = "/tmp/monmap";
 221
 222             eval {
 223                 mkdir $mondir;
 224
 225                 run_command("chown ceph:ceph $mondir");
 226
 227                 if ($moncount > 0) {
 228                     my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 229                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
 230                     file_set_contents($monmap, $mapdata);
 231                 } else {
 232                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
 233                 }
 234
 235                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
 236                 run_command("chown ceph:ceph -R $mondir");
 237             };
 238             my $err = $@;
 239             unlink $monmap;
 240             if ($err) {
 241                 File::Path::remove_tree($mondir);
 242                 die $err;
 243             }
 244
 245             $cfg->{$monsection} = {
 246                 'host' => $monname,
 247                 'mon addr' => $monaddr,
 248             };
 249
 250             cfs_write_file('ceph.conf', $cfg);
 251
 252             my $create_keys_pid = fork();
 253             if (!defined($create_keys_pid)) {
 254                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
 255             } elsif ($create_keys_pid == 0) {
 256                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
 257             } else {
 258                 PVE::Ceph::Services::ceph_service_cmd('start', $monsection);
 259
 260                 # to ensure we have the correct startup order.
 261                 eval { run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]) };
 262                 warn "Enable ceph-mon\@${monid}.service failed, do manually: $@\n" if $@;
 263                 waitpid($create_keys_pid, 0);
 264             }
 265
 266             # create manager
 267             if (!$param->{'exclude-manager'}) {
 268                 my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 269                 PVE::Ceph::Services::create_mgr($monid, $rados);
 270             }
 271             PVE::Ceph::Services::broadcast_ceph_services();
 272         };
 273
 274         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
 275     }});
 276
 277 __PACKAGE__->register_method ({
 278     name => 'destroymon',
 279     path => '{monid}',
 280     method => 'DELETE',
 281     description => "Destroy Ceph Monitor and Manager.",
 282     proxyto => 'node',
 283     protected => 1,
 284     permissions => {
 285         check => ['perm', '/', [ 'Sys.Modify' ]],
 286     },
 287     parameters => {
 288         additionalProperties => 0,
 289         properties => {
 290             node => get_standard_option('pve-node'),
 291             monid => {
 292                 description => 'Monitor ID',
 293                 type => 'string',
 294                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
 295             },
 296             'exclude-manager' => {
 297                 type => 'boolean',
 298                 default => 0,
 299                 optional => 1,
 300                 description => "When set, removes only the monitor, not the manager"
 301             }
 302         },
 303     },
 304     returns => { type => 'string' },
 305     code => sub {
 306         my ($param) = @_;
 307
 308         my $rpcenv = PVE::RPCEnvironment::get();
 309
 310         my $authuser = $rpcenv->get_user();
 311
 312         PVE::Ceph::Tools::check_ceph_inited();
 313
 314         my $cfg = cfs_read_file('ceph.conf');
 315
 316         my $monid = $param->{monid};
 317         my $monsection = "mon.$monid";
 318
 319         my $rados = PVE::RADOS->new();
 320         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 321         my $monlist = $monstat->{monmap}->{mons};
 322
 323         die "no such monitor id '$monid'\n"
 324             if !defined($cfg->{$monsection});
 325
 326         my $ccname = PVE::Ceph::Tools::get_config('ccname');
 327
 328         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 329         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
 330
 331         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
 332
 333         my $worker = sub {
 334             my $upid = shift;
 335
 336             # reopen with longer timeout
 337             $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 338
 339             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
 340
 341             eval { PVE::Ceph::Services::ceph_service_cmd('stop', $monsection); };
 342             warn $@ if $@;
 343
 344             delete $cfg->{$monsection};
 345             cfs_write_file('ceph.conf', $cfg);
 346             File::Path::remove_tree($mondir);
 347
 348             # remove manager
 349             if (!$param->{'exclude-manager'}) {
 350                 eval { PVE::Ceph::Services::destroy_mgr($monid) };
 351                 warn $@ if $@;
 352             }
 353             PVE::Ceph::Services::broadcast_ceph_services();
 354         };
 355
 356         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
 357     }});
 358
 359 1;