PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5 use File::Basename;
   6 use File::Path;
   7 use POSIX qw (LONG_MAX);
   8 use Cwd qw(abs_path);
   9 use IO::Dir;
  10 use UUID;
  11 use Net::IP;
  12
  13 use PVE::SafeSyslog;
  14 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  15 use PVE::Exception qw(raise raise_param_exc);
  16 use PVE::INotify;
  17 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
  18 use PVE::AccessControl;
  19 use PVE::Storage;
  20 use PVE::RESTHandler;
  21 use PVE::RPCEnvironment;
  22 use PVE::JSONSchema qw(get_standard_option);
  23 use JSON;
  24
  25 use base qw(PVE::RESTHandler);
  26
  27 use Data::Dumper; # fixme: remove
  28
  29 my $ccname = 'ceph'; # ceph cluster name
  30 my $ceph_cfgdir = "/etc/ceph";
  31 my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
  32 my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
  33 my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
  34 my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
  35
  36 my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
  37 my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
  38
  39 my $ceph_bin = "/usr/bin/ceph";
  40
  41 sub purge_all_ceph_files {
  42     # fixme: this is very dangerous - should we really support this function?
  43
  44     unlink $ceph_cfgpath;
  45
  46     unlink $pve_ceph_cfgpath;
  47     unlink $pve_ckeyring_path;
  48     unlink $pve_mon_key_path;
  49
  50     unlink $ceph_bootstrap_osd_keyring;
  51     unlink $ceph_bootstrap_mds_keyring;
  52
  53     system("rm -rf /var/lib/ceph/mon/ceph-*");
  54
  55     # remove osd?
  56
  57 }
  58
  59 my $check_ceph_installed = sub {
  60     my ($noerr) = @_;
  61
  62     if (! -x $ceph_bin) {
  63         die "ceph binaries not installed\n" if !$noerr;
  64         return undef;
  65     }
  66
  67     return 1;
  68 };
  69
  70 my $check_ceph_inited = sub {
  71     my ($noerr) = @_;
  72
  73     return undef if !&$check_ceph_installed($noerr);
  74
  75     if (! -f $pve_ceph_cfgpath) {
  76         die "pveceph configuration not initialized\n" if !$noerr;
  77         return undef;
  78     }
  79
  80     return 1;
  81 };
  82
  83 my $check_ceph_enabled = sub {
  84     my ($noerr) = @_;
  85
  86     return undef if !&$check_ceph_inited($noerr);
  87
  88     if (! -f $ceph_cfgpath) {
  89         die "pveceph configuration not enabled\n" if !$noerr;
  90         return undef;
  91     }
  92
  93     return 1;
  94 };
  95
  96 my $parse_ceph_config = sub {
  97     my ($filename) = @_;
  98
  99     my $cfg = {};
 100
 101     return $cfg if ! -f $filename;
 102
 103     my $fh = IO::File->new($filename, "r") ||
 104         die "unable to open '$filename' - $!\n";
 105
 106     my $section;
 107
 108     while (defined(my $line = <$fh>)) {
 109         $line =~ s/[;#].*$//;
 110         $line =~ s/^\s+//;
 111         $line =~ s/\s+$//;
 112         next if !$line;
 113
 114         $section = $1 if $line =~ m/^\[(\S+)\]$/;
 115         if (!$section) {
 116             warn "no section - skip: $line\n";
 117             next;
 118         }
 119
 120         if ($line =~ m/^(.*\S)\s*=\s*(\S.*)$/) {
 121             $cfg->{$section}->{$1} = $2;
 122         }
 123
 124     }
 125
 126     return $cfg;
 127 };
 128
 129 my $run_ceph_cmd = sub {
 130     my ($cmd, %params) = @_;
 131
 132     my $timeout = 5;
 133
 134     run_command(['ceph', '-c', $pve_ceph_cfgpath,
 135                  '--connect-timeout', $timeout,
 136                  @$cmd], %params);
 137 };
 138
 139 my $run_ceph_cmd_text = sub {
 140     my ($cmd, %opts) = @_;
 141
 142     my $out = '';
 143
 144     my $quiet = delete $opts{quiet};
 145
 146     my $parser = sub {
 147         my $line = shift;
 148         $out .= "$line\n";
 149     };
 150
 151     my $errfunc = sub {
 152         my $line = shift;
 153         print "$line\n" if !$quiet;
 154     };
 155
 156     &$run_ceph_cmd($cmd, outfunc => $parser, errfunc => $errfunc);
 157
 158     return $out;
 159 };
 160
 161 my $run_ceph_cmd_json = sub {
 162     my ($cmd, %opts) = @_;
 163
 164     my $json = &$run_ceph_cmd_text([@$cmd, '--format', 'json'], %opts);
 165
 166     return decode_json($json);
 167 };
 168
 169 sub ceph_mon_status {
 170     my ($quiet) = @_;
 171
 172     return &$run_ceph_cmd_json(['mon_status'], quiet => $quiet);
 173
 174 }
 175
 176 my $ceph_osd_status = sub {
 177     my ($quiet) = @_;
 178
 179     return &$run_ceph_cmd_json(['osd', 'dump'], quiet => $quiet);
 180 };
 181
 182 my $write_ceph_config = sub {
 183     my ($cfg) = @_;
 184
 185     my $out = '';
 186
 187     my $cond_write_sec = sub {
 188         my $re = shift;
 189
 190         foreach my $section (keys %$cfg) {
 191             next if $section !~ m/^$re$/;
 192             $out .= "[$section]\n";
 193             foreach my $key (sort keys %{$cfg->{$section}}) {
 194                 $out .= "\t $key = $cfg->{$section}->{$key}\n";
 195             }
 196             $out .= "\n";
 197         }
 198     };
 199
 200     &$cond_write_sec('global');
 201     &$cond_write_sec('mon');
 202     &$cond_write_sec('osd');
 203     &$cond_write_sec('mon\..*');
 204     &$cond_write_sec('osd\..*');
 205
 206     PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
 207 };
 208
 209 my $setup_pve_symlinks = sub {
 210     # fail if we find a real file instead of a link
 211     if (-f $ceph_cfgpath) {
 212         my $lnk = readlink($ceph_cfgpath);
 213         die "file '$ceph_cfgpath' already exists\n"
 214             if !$lnk || $lnk ne $pve_ceph_cfgpath;
 215     } else {
 216         symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
 217             die "unable to create symlink '$ceph_cfgpath' - $!\n";
 218     }
 219 };
 220
 221 my $ceph_service_cmd = sub {
 222     run_command(['service', 'ceph', '-c', $pve_ceph_cfgpath, @_]);
 223 };
 224
 225
 226 sub list_disks {
 227     my $disklist = {};
 228
 229     my $fd = IO::File->new("/proc/mounts", "r") ||
 230         die "unable to open /proc/mounts - $!\n";
 231
 232     my $mounted = {};
 233
 234     while (defined(my $line = <$fd>)) {
 235         my ($dev, $path, $fstype) = split(/\s+/, $line);
 236         next if !($dev && $path && $fstype);
 237         next if $dev !~ m|^/dev/|;
 238         my $real_dev = abs_path($dev);
 239         $mounted->{$real_dev} = $path;
 240     }
 241     close($fd);
 242
 243     my $dev_is_mounted = sub {
 244         my ($dev) = @_;
 245         return $mounted->{$dev};
 246     };
 247
 248     my $dir_is_epmty = sub {
 249         my ($dir) = @_;
 250
 251         my $dh = IO::Dir->new ($dir);
 252         return 1 if !$dh;
 253
 254         while (defined(my $tmp = $dh->read)) {
 255             next if $tmp eq '.' || $tmp eq '..';
 256             $dh->close;
 257             return 0;
 258         }
 259         $dh->close;
 260         return 1;
 261     };
 262
 263     dir_glob_foreach('/sys/block', '.*', sub {
 264         my ($dev) = @_;
 265
 266         return if $dev eq '.';
 267         return if $dev eq '..';
 268
 269         return if $dev =~ m|^ram\d+$|; # skip ram devices
 270         return if $dev =~ m|^loop\d+$|; # skip loop devices
 271         return if $dev =~ m|^md\d+$|; # skip md devices
 272         return if $dev =~ m|^dm-.*$|; # skip dm related things
 273         return if $dev =~ m|^fd\d+$|; # skip Floppy
 274         return if $dev =~ m|^sr\d+$|; # skip CDs
 275
 276         my $devdir = "/sys/block/$dev/device";
 277         return if ! -d $devdir;
 278
 279         my $size = file_read_firstline("/sys/block/$dev/size");
 280         return if !$size;
 281
 282         $size = $size * 512;
 283
 284         my $info = `udevadm info --path /sys/block/$dev --query all`;
 285         return if !$info;
 286
 287         return if $info !~ m/^E: DEVTYPE=disk$/m;
 288         return if $info =~ m/^E: ID_CDROM/m;
 289
 290         my $serial = 'unknown';
 291         if ($info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m) {
 292             $serial = $1;
 293         }
 294
 295         my $vendor = file_read_firstline("$devdir/vendor") || 'unknown';
 296         my $model = file_read_firstline("$devdir/model") || 'unknown';
 297
 298         my $used = &$dir_is_epmty("/sys/block/$dev/holders") ? 0 : 1;
 299
 300         $used = 1 if &$dev_is_mounted("/dev/$dev");
 301
 302         $disklist->{$dev} = {
 303             vendor => $vendor,
 304             model => $model,
 305             size => $size,
 306             serial => $serial,
 307         };
 308
 309         my $osdid = -1;
 310
 311         dir_glob_foreach("/sys/block/$dev", "$dev.+", sub {
 312             my ($part) = @_;
 313             if (!&$dir_is_epmty("/sys/block/$dev/$part/holders"))  {
 314                 $used = 1;
 315             }
 316             if (my $mp = &$dev_is_mounted("/dev/$part")) {
 317                 $used = 1;
 318                 if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) {
 319                     $osdid = $1;
 320                 }
 321             }
 322         });
 323
 324         $disklist->{$dev}->{used} = $used;
 325         $disklist->{$dev}->{osdid} = $osdid;
 326    });
 327
 328     return $disklist;
 329 }
 330
 331 __PACKAGE__->register_method ({
 332     name => 'index',
 333     path => '',
 334     method => 'GET',
 335     description => "Directory index.",
 336     permissions => { user => 'all' },
 337     parameters => {
 338         additionalProperties => 0,
 339         properties => {
 340             node => get_standard_option('pve-node'),
 341         },
 342     },
 343     returns => {
 344         type => 'array',
 345         items => {
 346             type => "object",
 347             properties => {},
 348         },
 349         links => [ { rel => 'child', href => "{name}" } ],
 350     },
 351     code => sub {
 352         my ($param) = @_;
 353
 354         my $result = [
 355             { name => 'init' },
 356             { name => 'mon' },
 357             { name => 'osd' },
 358             { name => 'pools' },
 359             { name => 'stop' },
 360             { name => 'start' },
 361             { name => 'status' },
 362             { name => 'crush' },
 363             { name => 'config' },
 364             { name => 'log' },
 365             { name => 'disks' },
 366         ];
 367
 368         return $result;
 369     }});
 370
 371 __PACKAGE__->register_method ({
 372     name => 'disks',
 373     path => 'disks',
 374     method => 'GET',
 375     description => "List local disks.",
 376     proxyto => 'node',
 377     protected => 1,
 378     parameters => {
 379         additionalProperties => 0,
 380         properties => {
 381             node => get_standard_option('pve-node'),
 382         },
 383     },
 384     returns => {
 385         type => 'array',
 386         items => {
 387             type => "object",
 388             properties => {
 389                 dev => { type => 'string' },
 390                 used => { type => 'boolean' },
 391                 size => { type => 'integer' },
 392                 osdid => { type => 'integer' },
 393                 vendor =>  { type => 'string', optional => 1 },
 394                 model =>  { type => 'string', optional => 1 },
 395                 serial =>  { type => 'string', optional => 1 },
 396             },
 397         },
 398         # links => [ { rel => 'child', href => "{}" } ],
 399     },
 400     code => sub {
 401         my ($param) = @_;
 402
 403         &$check_ceph_inited();
 404
 405         my $res = list_disks();
 406
 407         return PVE::RESTHandler::hash_to_array($res, 'dev');
 408     }});
 409
 410 __PACKAGE__->register_method ({
 411     name => 'config',
 412     path => 'config',
 413     method => 'GET',
 414     description => "Get Ceph configuration.",
 415     parameters => {
 416         additionalProperties => 0,
 417         properties => {
 418             node => get_standard_option('pve-node'),
 419         },
 420     },
 421     returns => { type => 'string' },
 422     code => sub {
 423         my ($param) = @_;
 424
 425         &$check_ceph_inited();
 426
 427         return PVE::Tools::file_get_contents($pve_ceph_cfgpath);
 428
 429     }});
 430
 431 __PACKAGE__->register_method ({
 432     name => 'listmon',
 433     path => 'mon',
 434     method => 'GET',
 435     description => "Get Ceph monitor list.",
 436     proxyto => 'node',
 437     protected => 1,
 438     parameters => {
 439         additionalProperties => 0,
 440         properties => {
 441             node => get_standard_option('pve-node'),
 442         },
 443     },
 444     returns => {
 445         type => 'array',
 446         items => {
 447             type => "object",
 448             properties => {
 449                 name => { type => 'string' },
 450                 addr => { type => 'string' },
 451             },
 452         },
 453         links => [ { rel => 'child', href => "{name}" } ],
 454     },
 455     code => sub {
 456         my ($param) = @_;
 457
 458         &$check_ceph_inited();
 459
 460         my $res = [];
 461
 462         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 463
 464         my $monhash = {};
 465         foreach my $section (keys %$cfg) {
 466             my $d = $cfg->{$section};
 467             if ($section =~ m/^mon\.(\S+)$/) {
 468                 my $monid = $1;
 469                 if ($d->{'mon addr'} && $d->{'host'}) {
 470                     $monhash->{$monid} = {
 471                         addr => $d->{'mon addr'},
 472                         host => $d->{'host'},
 473                         name => $monid,
 474                     }
 475                 }
 476             }
 477         }
 478
 479         eval {
 480             my $monstat = ceph_mon_status();
 481             my $mons = $monstat->{monmap}->{mons};
 482             foreach my $d (@$mons) {
 483                 next if !defined($d->{name});
 484                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 485                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 486                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 487                     $monhash->{$d->{name}}->{quorum} = 1;
 488                 }
 489             }
 490         };
 491         warn $@ if $@;
 492
 493         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 494     }});
 495
 496 __PACKAGE__->register_method ({
 497     name => 'init',
 498     path => 'init',
 499     method => 'POST',
 500     description => "Create initial ceph default configuration and setup symlinks.",
 501     proxyto => 'node',
 502     protected => 1,
 503     parameters => {
 504         additionalProperties => 0,
 505         properties => {
 506             node => get_standard_option('pve-node'),
 507             network => {
 508                 description => "Use specific network for all ceph related traffic",
 509                 type => 'string', format => 'CIDR',
 510                 optional => 1,
 511                 maxLength => 128,
 512             },
 513             size => {
 514                 description => 'Number of replicas per object',
 515                 type => 'integer',
 516                 default => 2,
 517                 optional => 1,
 518                 minimum => 1,
 519                 maximum => 3,
 520             },
 521             pg_bits => {
 522                 description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)",
 523                 type => 'integer',
 524                 default => 6,
 525                 optional => 1,
 526                 minimum => 6,
 527                 maximum => 14,
 528             },
 529         },
 530     },
 531     returns => { type => 'null' },
 532     code => sub {
 533         my ($param) = @_;
 534
 535         &$check_ceph_installed();
 536
 537         # simply load old config if it already exists
 538         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 539
 540         if (!$cfg->{global}) {
 541
 542             my $fsid;
 543             my $uuid;
 544
 545             UUID::generate($uuid);
 546             UUID::unparse($uuid, $fsid);
 547
 548             $cfg->{global} = {
 549                 'fsid' => $fsid,
 550                 'auth supported' => 'cephx',
 551                 'auth cluster required' => 'cephx',
 552                 'auth service required' => 'cephx',
 553                 'auth client required' => 'cephx',
 554                 'filestore xattr use omap' => 'true',
 555                 'osd journal size' => '1024',
 556                 'osd pool default min size' => 1,
 557             };
 558
 559             # this does not work for default pools
 560             #'osd pool default pg num' => $pg_num,
 561             #'osd pool default pgp num' => $pg_num,
 562         }
 563
 564         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 565         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 566
 567         $cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size};
 568
 569         if ($param->{pg_bits}) {
 570             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 571             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 572         }
 573
 574         if ($param->{network}) {
 575             $cfg->{global}->{'public network'} = $param->{network};
 576             $cfg->{global}->{'cluster network'} = $param->{network};
 577         }
 578
 579         &$write_ceph_config($cfg);
 580
 581         &$setup_pve_symlinks();
 582
 583         return undef;
 584     }});
 585
 586 my $find_node_ip = sub {
 587     my ($cidr) = @_;
 588
 589     my $config = PVE::INotify::read_file('interfaces');
 590
 591     my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n";
 592
 593     foreach my $iface (keys %$config) {
 594         my $d = $config->{$iface};
 595         next if !$d->{address};
 596         my $a = Net::IP->new($d->{address});
 597         next if !$a;
 598         return $d->{address} if $net->overlaps($a);
 599     }
 600
 601     die "unable to find local address within network '$cidr'\n";
 602 };
 603
 604 __PACKAGE__->register_method ({
 605     name => 'createmon',
 606     path => 'mon',
 607     method => 'POST',
 608     description => "Create Ceph Monitor",
 609     proxyto => 'node',
 610     protected => 1,
 611     parameters => {
 612         additionalProperties => 0,
 613         properties => {
 614             node => get_standard_option('pve-node'),
 615         },
 616     },
 617     returns => { type => 'string' },
 618     code => sub {
 619         my ($param) = @_;
 620
 621         &$check_ceph_inited();
 622
 623         &$setup_pve_symlinks();
 624
 625         my $rpcenv = PVE::RPCEnvironment::get();
 626
 627         my $authuser = $rpcenv->get_user();
 628
 629         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 630
 631         my $moncount = 0;
 632
 633         my $monaddrhash = {};
 634
 635         foreach my $section (keys %$cfg) {
 636             next if $section eq 'global';
 637             my $d = $cfg->{$section};
 638             if ($section =~ m/^mon\./) {
 639                 $moncount++;
 640                 if ($d->{'mon addr'}) {
 641                     $monaddrhash->{$d->{'mon addr'}} = $section;
 642                 }
 643             }
 644         }
 645
 646         my $monid;
 647         for (my $i = 0; $i < 7; $i++) {
 648             if (!$cfg->{"mon.$i"}) {
 649                 $monid = $i;
 650                 last;
 651             }
 652         }
 653         die "unable to find usable monitor id\n" if !defined($monid);
 654
 655         my $monsection = "mon.$monid";
 656         my $ip;
 657         if (my $pubnet = $cfg->{global}->{'public network'}) {
 658             $ip = &$find_node_ip($pubnet);
 659         } else {
 660             $ip = PVE::Cluster::remote_node_ip($param->{node});
 661         }
 662
 663         my $monaddr = "$ip:6789";
 664         my $monname = $param->{node};
 665
 666         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
 667         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
 668             if $monaddrhash->{$monaddr};
 669
 670         my $worker = sub  {
 671             my $upid = shift;
 672
 673             if (! -f $pve_ckeyring_path) {
 674                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
 675                             "--gen-key -n client.admin");
 676             }
 677
 678             if (! -f $pve_mon_key_path) {
 679                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
 680                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
 681                             "--cap mds 'allow' " .
 682                             "--cap osd 'allow *' " .
 683                             "--cap mon 'allow *'");
 684                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
 685                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
 686             }
 687
 688             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 689             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
 690
 691             my $monmap = "/tmp/monmap";
 692
 693             eval {
 694                 mkdir $mondir;
 695
 696                 if ($moncount > 0) {
 697                     my $monstat = ceph_mon_status(); # online test
 698                     &$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]);
 699                 } else {
 700                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
 701                 }
 702
 703                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
 704             };
 705             my $err = $@;
 706             unlink $monmap;
 707             if ($err) {
 708                 File::Path::remove_tree($mondir);
 709                 die $err;
 710             }
 711
 712             $cfg->{$monsection} = {
 713                 'host' => $monname,
 714                 'mon addr' => $monaddr,
 715             };
 716
 717             &$write_ceph_config($cfg);
 718
 719             &$ceph_service_cmd('start', $monsection);
 720         };
 721
 722         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
 723     }});
 724
 725 __PACKAGE__->register_method ({
 726     name => 'destroymon',
 727     path => 'mon/{monid}',
 728     method => 'DELETE',
 729     description => "Destroy Ceph monitor.",
 730     proxyto => 'node',
 731     protected => 1,
 732     parameters => {
 733         additionalProperties => 0,
 734         properties => {
 735             node => get_standard_option('pve-node'),
 736             monid => {
 737                 description => 'Monitor ID',
 738                 type => 'integer',
 739             },
 740         },
 741     },
 742     returns => { type => 'string' },
 743     code => sub {
 744         my ($param) = @_;
 745
 746         my $rpcenv = PVE::RPCEnvironment::get();
 747
 748         my $authuser = $rpcenv->get_user();
 749
 750         &$check_ceph_inited();
 751
 752         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 753
 754         my $monid = $param->{monid};
 755         my $monsection = "mon.$monid";
 756
 757         my $monstat = ceph_mon_status();
 758         my $monlist = $monstat->{monmap}->{mons};
 759
 760         die "no such monitor id '$monid'\n"
 761             if !defined($cfg->{$monsection});
 762
 763
 764         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 765         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
 766
 767         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
 768
 769         my $worker = sub {
 770             my $upid = shift;
 771
 772             &$run_ceph_cmd(['mon', 'remove', $monid]);
 773
 774             eval { &$ceph_service_cmd('stop', $monsection); };
 775             warn $@ if $@;
 776
 777             delete $cfg->{$monsection};
 778             &$write_ceph_config($cfg);
 779             File::Path::remove_tree($mondir);
 780         };
 781
 782         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
 783     }});
 784
 785 __PACKAGE__->register_method ({
 786     name => 'stop',
 787     path => 'stop',
 788     method => 'POST',
 789     description => "Stop ceph services.",
 790     proxyto => 'node',
 791     protected => 1,
 792     parameters => {
 793         additionalProperties => 0,
 794         properties => {
 795             node => get_standard_option('pve-node'),
 796             service => {
 797                 description => 'Ceph service name.',
 798                 type => 'string',
 799                 optional => 1,
 800                 pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
 801             },
 802         },
 803     },
 804     returns => { type => 'string' },
 805     code => sub {
 806         my ($param) = @_;
 807
 808         my $rpcenv = PVE::RPCEnvironment::get();
 809
 810         my $authuser = $rpcenv->get_user();
 811
 812         &$check_ceph_inited();
 813
 814         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 815         scalar(keys %$cfg) || die "no configuration\n";
 816
 817         my $worker = sub {
 818             my $upid = shift;
 819
 820             my $cmd = ['stop'];
 821             if ($param->{service}) {
 822                 push @$cmd, $param->{service};
 823             }
 824
 825             &$ceph_service_cmd(@$cmd);
 826         };
 827
 828         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 829                                     $authuser, $worker);
 830     }});
 831
 832 __PACKAGE__->register_method ({
 833     name => 'start',
 834     path => 'start',
 835     method => 'POST',
 836     description => "Start ceph services.",
 837     proxyto => 'node',
 838     protected => 1,
 839     parameters => {
 840         additionalProperties => 0,
 841         properties => {
 842             node => get_standard_option('pve-node'),
 843             service => {
 844                 description => 'Ceph service name.',
 845                 type => 'string',
 846                 optional => 1,
 847                 pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
 848             },
 849         },
 850     },
 851     returns => { type => 'string' },
 852     code => sub {
 853         my ($param) = @_;
 854
 855         my $rpcenv = PVE::RPCEnvironment::get();
 856
 857         my $authuser = $rpcenv->get_user();
 858
 859         &$check_ceph_inited();
 860
 861         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 862         scalar(keys %$cfg) || die "no configuration\n";
 863
 864         my $worker = sub {
 865             my $upid = shift;
 866
 867             my $cmd = ['start'];
 868             if ($param->{service}) {
 869                 push @$cmd, $param->{service};
 870             }
 871
 872             &$ceph_service_cmd(@$cmd);
 873         };
 874
 875         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 876                                     $authuser, $worker);
 877     }});
 878
 879 __PACKAGE__->register_method ({
 880     name => 'status',
 881     path => 'status',
 882     method => 'GET',
 883     description => "Get ceph status.",
 884     proxyto => 'node',
 885     protected => 1,
 886     parameters => {
 887         additionalProperties => 0,
 888         properties => {
 889             node => get_standard_option('pve-node'),
 890         },
 891     },
 892     returns => { type => 'object' },
 893     code => sub {
 894         my ($param) = @_;
 895
 896         &$check_ceph_enabled();
 897
 898         return &$run_ceph_cmd_json(['status'], quiet => 1);
 899     }});
 900
 901 __PACKAGE__->register_method ({
 902     name => 'lspools',
 903     path => 'pools',
 904     method => 'GET',
 905     description => "List all pools.",
 906     proxyto => 'node',
 907     protected => 1,
 908     parameters => {
 909         additionalProperties => 0,
 910         properties => {
 911             node => get_standard_option('pve-node'),
 912         },
 913     },
 914     returns => {
 915         type => 'array',
 916         items => {
 917             type => "object",
 918             properties => {
 919                 pool => { type => 'integer' },
 920                 pool_name => { type => 'string' },
 921                 size => { type => 'integer' },
 922             },
 923         },
 924         links => [ { rel => 'child', href => "{pool_name}" } ],
 925     },
 926     code => sub {
 927         my ($param) = @_;
 928
 929         &$check_ceph_inited();
 930
 931         my $res = &$run_ceph_cmd_json(['osd', 'dump'], quiet => 1);
 932
 933         my $data = [];
 934         foreach my $e (@{$res->{pools}}) {
 935             my $d = {};
 936             foreach my $attr (qw(pool pool_name size min_size pg_num crush_ruleset)) {
 937                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
 938             }
 939             push @$data, $d;
 940         }
 941
 942         return $data;
 943     }});
 944
 945 __PACKAGE__->register_method ({
 946     name => 'createpool',
 947     path => 'pools',
 948     method => 'POST',
 949     description => "Create POOL",
 950     proxyto => 'node',
 951     protected => 1,
 952     parameters => {
 953         additionalProperties => 0,
 954         properties => {
 955             node => get_standard_option('pve-node'),
 956             name => {
 957                 description => "The name of the pool. It must be unique.",
 958                 type => 'string',
 959             },
 960             size => {
 961                 description => 'Number of replicas per object',
 962                 type => 'integer',
 963                 default => 2,
 964                 optional => 1,
 965                 minimum => 1,
 966                 maximum => 3,
 967             },
 968             min_size => {
 969                 description => 'Minimum number of replicas per object',
 970                 type => 'integer',
 971                 default => 1,
 972                 optional => 1,
 973                 minimum => 1,
 974                 maximum => 3,
 975             },
 976             pg_num => {
 977                 description => "Number of placement groups.",
 978                 type => 'integer',
 979                 default => 512,
 980                 optional => 1,
 981                 minimum => 8,
 982                 maximum => 32768,
 983             },
 984         },
 985     },
 986     returns => { type => 'null' },
 987     code => sub {
 988         my ($param) = @_;
 989
 990         &$check_ceph_inited();
 991
 992         die "not fully configured - missing '$pve_ckeyring_path'\n"
 993             if ! -f $pve_ckeyring_path;
 994
 995         my $pg_num = $param->{pg_num} || 512;
 996         my $size = $param->{size} || 2;
 997         my $min_size = $param->{min_size} || 1;
 998
 999         &$run_ceph_cmd(['osd', 'pool', 'create', $param->{name}, $pg_num]);
1000
1001         &$run_ceph_cmd(['osd', 'pool', 'set', $param->{name}, 'min_size', $min_size]);
1002
1003         &$run_ceph_cmd(['osd', 'pool', 'set', $param->{name}, 'size', $size]);
1004
1005         return undef;
1006     }});
1007
1008 __PACKAGE__->register_method ({
1009     name => 'destroypool',
1010     path => 'pools/{name}',
1011     method => 'DELETE',
1012     description => "Destroy pool",
1013     proxyto => 'node',
1014     protected => 1,
1015     parameters => {
1016         additionalProperties => 0,
1017         properties => {
1018             node => get_standard_option('pve-node'),
1019             name => {
1020                 description => "The name of the pool. It must be unique.",
1021                 type => 'string',
1022             },
1023         },
1024     },
1025     returns => { type => 'null' },
1026     code => sub {
1027         my ($param) = @_;
1028
1029         &$check_ceph_inited();
1030
1031         &$run_ceph_cmd(['osd', 'pool', 'delete', $param->{name}, $param->{name}, '--yes-i-really-really-mean-it']);
1032
1033         return undef;
1034     }});
1035
1036 __PACKAGE__->register_method ({
1037     name => 'listosd',
1038     path => 'osd',
1039     method => 'GET',
1040     description => "Get Ceph osd list/tree.",
1041     proxyto => 'node',
1042     protected => 1,
1043     parameters => {
1044         additionalProperties => 0,
1045         properties => {
1046             node => get_standard_option('pve-node'),
1047         },
1048     },
1049     returns => {
1050         type => "object",
1051     },
1052     code => sub {
1053         my ($param) = @_;
1054
1055         &$check_ceph_inited();
1056
1057         my $res = &$run_ceph_cmd_json(['osd', 'tree'], quiet => 1);
1058
1059         die "no tree nodes found\n" if !($res && $res->{nodes});
1060
1061         my $nodes = {};
1062         my $newnodes = {};
1063         foreach my $e (@{$res->{nodes}}) {
1064             $nodes->{$e->{id}} = $e;
1065
1066             my $new = {
1067                 id => $e->{id},
1068                 name => $e->{name},
1069                 type => $e->{type}
1070             };
1071
1072             foreach my $opt (qw(status crush_weight reweight)) {
1073                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
1074             }
1075
1076             $newnodes->{$e->{id}} = $new;
1077         }
1078
1079         foreach my $e (@{$res->{nodes}}) {
1080             my $new = $newnodes->{$e->{id}};
1081             if ($e->{children} && scalar(@{$e->{children}})) {
1082                 $new->{children} = [];
1083                 $new->{leaf} = 0;
1084                 foreach my $cid (@{$e->{children}}) {
1085                     $nodes->{$cid}->{parent} = $e->{id};
1086                     if ($nodes->{$cid}->{type} eq 'osd' &&
1087                         $e->{type} eq 'host') {
1088                         $newnodes->{$cid}->{host} = $e->{name};
1089                     }
1090                     push @{$new->{children}}, $newnodes->{$cid};
1091                 }
1092             } else {
1093                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
1094             }
1095         }
1096
1097         my $rootnode;
1098         foreach my $e (@{$res->{nodes}}) {
1099             if (!$nodes->{$e->{id}}->{parent}) {
1100                 $rootnode = $newnodes->{$e->{id}};
1101                 last;
1102             }
1103         }
1104
1105         die "no root node\n" if !$rootnode;
1106
1107         my $data = { root => $rootnode };
1108
1109         return $data;
1110     }});
1111
1112 __PACKAGE__->register_method ({
1113     name => 'createosd',
1114     path => 'osd',
1115     method => 'POST',
1116     description => "Create OSD",
1117     proxyto => 'node',
1118     protected => 1,
1119     parameters => {
1120         additionalProperties => 0,
1121         properties => {
1122             node => get_standard_option('pve-node'),
1123             dev => {
1124                 description => "Block device name.",
1125                 type => 'string',
1126             },
1127             fstype => {
1128                 description => "File system type.",
1129                 type => 'string',
1130                 enum => ['xfs', 'ext4', 'btrfs'],
1131                 default => 'xfs',
1132                 optional => 1,
1133             },
1134         },
1135     },
1136     returns => { type => 'string' },
1137     code => sub {
1138         my ($param) = @_;
1139
1140         my $rpcenv = PVE::RPCEnvironment::get();
1141
1142         my $authuser = $rpcenv->get_user();
1143
1144         &$check_ceph_inited();
1145
1146         &$setup_pve_symlinks();
1147
1148         -b  $param->{dev} || die "no such block device '$param->{dev}'\n";
1149
1150         my $disklist = list_disks();
1151
1152         my $devname = $param->{dev};
1153         $devname =~ s|/dev/||;
1154
1155         my $diskinfo = $disklist->{$devname};
1156         die "unable to get device info for '$devname'\n"
1157             if !$diskinfo;
1158
1159         die "device '$param->{dev}' is in use\n"
1160             if $diskinfo->{used};
1161
1162         my $monstat = ceph_mon_status(1);
1163         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
1164         my $fsid = $monstat->{monmap}->{fsid};
1165
1166         if (! -f $ceph_bootstrap_osd_keyring) {
1167             &$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]);
1168         };
1169
1170         my $worker = sub {
1171             my $upid = shift;
1172
1173             my $fstype = $param->{fstype} || 'xfs';
1174
1175             print "create OSD on $param->{dev} ($fstype)\n";
1176
1177             run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', $fstype,
1178                          '--cluster', $ccname, '--cluster-uuid', $fsid,
1179                          '--', $param->{dev}]);
1180         };
1181
1182         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
1183     }});
1184
1185 __PACKAGE__->register_method ({
1186     name => 'destroyosd',
1187     path => 'osd/{osdid}',
1188     method => 'DELETE',
1189     description => "Destroy OSD",
1190     proxyto => 'node',
1191     protected => 1,
1192     parameters => {
1193         additionalProperties => 0,
1194         properties => {
1195             node => get_standard_option('pve-node'),
1196             osdid => {
1197                 description => 'OSD ID',
1198                 type => 'integer',
1199             },
1200         },
1201     },
1202     returns => { type => 'string' },
1203     code => sub {
1204         my ($param) = @_;
1205
1206         my $rpcenv = PVE::RPCEnvironment::get();
1207
1208         my $authuser = $rpcenv->get_user();
1209
1210         &$check_ceph_inited();
1211
1212         my $osdid = $param->{osdid};
1213
1214         # fixme: not 100% sure what we should do here
1215
1216         my $stat = &$ceph_osd_status();
1217
1218         my $osdlist = $stat->{osds} || [];
1219
1220         my $osdstat;
1221         foreach my $d (@$osdlist) {
1222             if ($d->{osd} == $osdid) {
1223                 $osdstat = $d;
1224                 last;
1225             }
1226         }
1227         die "no such OSD '$osdid'\n" if !$osdstat;
1228
1229         die "osd is in use (in == 1)\n" if $osdstat->{in};
1230         #&$run_ceph_cmd(['osd', 'out', $osdid]);
1231
1232         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
1233
1234         my $osdsection = "osd.$osdid";
1235
1236         my $worker = sub {
1237             my $upid = shift;
1238
1239             print "destroy OSD $osdsection\n";
1240
1241             eval { &$ceph_service_cmd('stop', $osdsection); };
1242             warn $@ if $@;
1243
1244             print "Remove $osdsection from the CRUSH map\n";
1245             &$run_ceph_cmd(['osd', 'crush', 'remove', $osdsection]);
1246
1247             print "Remove the $osdsection authentication key.\n";
1248             &$run_ceph_cmd(['auth', 'del', $osdsection]);
1249
1250             print "Remove OSD $osdsection\n";
1251             &$run_ceph_cmd(['osd', 'rm', $osdid]);
1252         };
1253
1254         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
1255     }});
1256
1257 __PACKAGE__->register_method ({
1258     name => 'crush',
1259     path => 'crush',
1260     method => 'GET',
1261     description => "Get OSD crush map",
1262     proxyto => 'node',
1263     protected => 1,
1264     parameters => {
1265         additionalProperties => 0,
1266         properties => {
1267             node => get_standard_option('pve-node'),
1268         },
1269     },
1270     returns => { type => 'string' },
1271     code => sub {
1272         my ($param) = @_;
1273
1274         &$check_ceph_inited();
1275
1276         # this produces JSON (difficult to read for the user)
1277         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1278
1279         my $txt = '';
1280
1281         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1282         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1283
1284         eval {
1285             &$run_ceph_cmd(['osd', 'getcrushmap', '-o', $mapfile]);
1286             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1287             $txt = PVE::Tools::file_get_contents($mapdata);
1288         };
1289         my $err = $@;
1290
1291         unlink $mapfile;
1292         unlink $mapdata;
1293
1294         die $err if $err;
1295
1296         return $txt;
1297     }});
1298
1299 __PACKAGE__->register_method({
1300     name => 'log',
1301     path => 'log',
1302     method => 'GET',
1303     description => "Read ceph log",
1304     proxyto => 'node',
1305     permissions => {
1306         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1307     },
1308     protected => 1,
1309     parameters => {
1310         additionalProperties => 0,
1311         properties => {
1312             node => get_standard_option('pve-node'),
1313             start => {
1314                 type => 'integer',
1315                 minimum => 0,
1316                 optional => 1,
1317             },
1318             limit => {
1319                 type => 'integer',
1320                 minimum => 0,
1321                 optional => 1,
1322             },
1323         },
1324     },
1325     returns => {
1326         type => 'array',
1327         items => {
1328             type => "object",
1329             properties => {
1330                 n => {
1331                   description=>  "Line number",
1332                   type=> 'integer',
1333                 },
1334                 t => {
1335                   description=>  "Line text",
1336                   type => 'string',
1337                 }
1338             }
1339         }
1340     },
1341     code => sub {
1342         my ($param) = @_;
1343
1344         my $rpcenv = PVE::RPCEnvironment::get();
1345         my $user = $rpcenv->get_user();
1346         my $node = $param->{node};
1347
1348         my $logfile = "/var/log/ceph/ceph.log";
1349         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1350
1351         $rpcenv->set_result_attrib('total', $count);
1352
1353         return $lines;
1354     }});
1355
1356