PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5 use File::Basename;
   6 use File::Path;
   7 use POSIX qw (LONG_MAX);
   8 use Cwd qw(abs_path);
   9 use IO::Dir;
  10 use UUID;
  11 use Net::IP;
  12
  13 use PVE::SafeSyslog;
  14 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  15 use PVE::Exception qw(raise raise_param_exc);
  16 use PVE::INotify;
  17 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
  18 use PVE::AccessControl;
  19 use PVE::Storage;
  20 use PVE::RESTHandler;
  21 use PVE::RPCEnvironment;
  22 use PVE::JSONSchema qw(get_standard_option);
  23 use JSON;
  24
  25 use base qw(PVE::RESTHandler);
  26
  27 use Data::Dumper; # fixme: remove
  28
  29 my $ccname = 'ceph'; # ceph cluster name
  30 my $ceph_cfgdir = "/etc/ceph";
  31 my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
  32 my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
  33 my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
  34 my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
  35
  36 my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
  37 my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
  38
  39 my $ceph_bin = "/usr/bin/ceph";
  40
  41 my $pve_osd_default_journal_size = 1024*5;
  42
  43 sub purge_all_ceph_files {
  44     # fixme: this is very dangerous - should we really support this function?
  45
  46     unlink $ceph_cfgpath;
  47
  48     unlink $pve_ceph_cfgpath;
  49     unlink $pve_ckeyring_path;
  50     unlink $pve_mon_key_path;
  51
  52     unlink $ceph_bootstrap_osd_keyring;
  53     unlink $ceph_bootstrap_mds_keyring;
  54
  55     system("rm -rf /var/lib/ceph/mon/ceph-*");
  56
  57     # remove osd?
  58
  59 }
  60
  61 my $check_ceph_installed = sub {
  62     my ($noerr) = @_;
  63
  64     if (! -x $ceph_bin) {
  65         die "ceph binaries not installed\n" if !$noerr;
  66         return undef;
  67     }
  68
  69     return 1;
  70 };
  71
  72 my $check_ceph_inited = sub {
  73     my ($noerr) = @_;
  74
  75     return undef if !&$check_ceph_installed($noerr);
  76
  77     if (! -f $pve_ceph_cfgpath) {
  78         die "pveceph configuration not initialized\n" if !$noerr;
  79         return undef;
  80     }
  81
  82     return 1;
  83 };
  84
  85 my $check_ceph_enabled = sub {
  86     my ($noerr) = @_;
  87
  88     return undef if !&$check_ceph_inited($noerr);
  89
  90     if (! -f $ceph_cfgpath) {
  91         die "pveceph configuration not enabled\n" if !$noerr;
  92         return undef;
  93     }
  94
  95     return 1;
  96 };
  97
  98 my $parse_ceph_config = sub {
  99     my ($filename) = @_;
 100
 101     my $cfg = {};
 102
 103     return $cfg if ! -f $filename;
 104
 105     my $fh = IO::File->new($filename, "r") ||
 106         die "unable to open '$filename' - $!\n";
 107
 108     my $section;
 109
 110     while (defined(my $line = <$fh>)) {
 111         $line =~ s/[;#].*$//;
 112         $line =~ s/^\s+//;
 113         $line =~ s/\s+$//;
 114         next if !$line;
 115
 116         $section = $1 if $line =~ m/^\[(\S+)\]$/;
 117         if (!$section) {
 118             warn "no section - skip: $line\n";
 119             next;
 120         }
 121
 122         if ($line =~ m/^(.*\S)\s*=\s*(\S.*)$/) {
 123             $cfg->{$section}->{$1} = $2;
 124         }
 125
 126     }
 127
 128     return $cfg;
 129 };
 130
 131 my $run_ceph_cmd = sub {
 132     my ($cmd, %params) = @_;
 133
 134     my $timeout = 5;
 135
 136     run_command(['ceph', '-c', $pve_ceph_cfgpath,
 137                  '--connect-timeout', $timeout,
 138                  @$cmd], %params);
 139 };
 140
 141 my $run_ceph_cmd_text = sub {
 142     my ($cmd, %opts) = @_;
 143
 144     my $out = '';
 145
 146     my $quiet = delete $opts{quiet};
 147
 148     my $parser = sub {
 149         my $line = shift;
 150         $out .= "$line\n";
 151     };
 152
 153     my $errfunc = sub {
 154         my $line = shift;
 155         print "$line\n" if !$quiet;
 156     };
 157
 158     &$run_ceph_cmd($cmd, outfunc => $parser, errfunc => $errfunc);
 159
 160     return $out;
 161 };
 162
 163 my $run_ceph_cmd_json = sub {
 164     my ($cmd, %opts) = @_;
 165
 166     my $json = &$run_ceph_cmd_text([@$cmd, '--format', 'json'], %opts);
 167
 168     return decode_json($json);
 169 };
 170
 171 sub ceph_mon_status {
 172     my ($quiet) = @_;
 173
 174     return &$run_ceph_cmd_json(['mon_status'], quiet => $quiet);
 175
 176 }
 177
 178 my $ceph_osd_status = sub {
 179     my ($quiet) = @_;
 180
 181     return &$run_ceph_cmd_json(['osd', 'dump'], quiet => $quiet);
 182 };
 183
 184 my $write_ceph_config = sub {
 185     my ($cfg) = @_;
 186
 187     my $out = '';
 188
 189     my $cond_write_sec = sub {
 190         my $re = shift;
 191
 192         foreach my $section (keys %$cfg) {
 193             next if $section !~ m/^$re$/;
 194             $out .= "[$section]\n";
 195             foreach my $key (sort keys %{$cfg->{$section}}) {
 196                 $out .= "\t $key = $cfg->{$section}->{$key}\n";
 197             }
 198             $out .= "\n";
 199         }
 200     };
 201
 202     &$cond_write_sec('global');
 203     &$cond_write_sec('mon');
 204     &$cond_write_sec('osd');
 205     &$cond_write_sec('mon\..*');
 206     &$cond_write_sec('osd\..*');
 207
 208     PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
 209 };
 210
 211 my $setup_pve_symlinks = sub {
 212     # fail if we find a real file instead of a link
 213     if (-f $ceph_cfgpath) {
 214         my $lnk = readlink($ceph_cfgpath);
 215         die "file '$ceph_cfgpath' already exists\n"
 216             if !$lnk || $lnk ne $pve_ceph_cfgpath;
 217     } else {
 218         symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
 219             die "unable to create symlink '$ceph_cfgpath' - $!\n";
 220     }
 221 };
 222
 223 my $ceph_service_cmd = sub {
 224     run_command(['service', 'ceph', '-c', $pve_ceph_cfgpath, @_]);
 225 };
 226
 227 sub list_disks {
 228     my $disklist = {};
 229
 230     my $fd = IO::File->new("/proc/mounts", "r") ||
 231         die "unable to open /proc/mounts - $!\n";
 232
 233     my $mounted = {};
 234
 235     while (defined(my $line = <$fd>)) {
 236         my ($dev, $path, $fstype) = split(/\s+/, $line);
 237         next if !($dev && $path && $fstype);
 238         next if $dev !~ m|^/dev/|;
 239         my $real_dev = abs_path($dev);
 240         $mounted->{$real_dev} = $path;
 241     }
 242     close($fd);
 243
 244     my $dev_is_mounted = sub {
 245         my ($dev) = @_;
 246         return $mounted->{$dev};
 247     };
 248
 249     my $dir_is_epmty = sub {
 250         my ($dir) = @_;
 251
 252         my $dh = IO::Dir->new ($dir);
 253         return 1 if !$dh;
 254
 255         while (defined(my $tmp = $dh->read)) {
 256             next if $tmp eq '.' || $tmp eq '..';
 257             $dh->close;
 258             return 0;
 259         }
 260         $dh->close;
 261         return 1;
 262     };
 263
 264     my $journal_uuid = '45b0969e-9b03-4f30-b4c6-b4b80ceff106';
 265
 266     my $journalhash = {};
 267     dir_glob_foreach('/dev/disk/by-parttypeuuid', "$journal_uuid\..+", sub {
 268         my ($entry) = @_;
 269         my $real_dev = abs_path("/dev/disk/by-parttypeuuid/$entry");
 270         $journalhash->{$real_dev} = 1;
 271     });
 272
 273     dir_glob_foreach('/sys/block', '.*', sub {
 274         my ($dev) = @_;
 275
 276         return if $dev eq '.';
 277         return if $dev eq '..';
 278
 279         return if $dev =~ m|^ram\d+$|; # skip ram devices
 280         return if $dev =~ m|^loop\d+$|; # skip loop devices
 281         return if $dev =~ m|^md\d+$|; # skip md devices
 282         return if $dev =~ m|^dm-.*$|; # skip dm related things
 283         return if $dev =~ m|^fd\d+$|; # skip Floppy
 284         return if $dev =~ m|^sr\d+$|; # skip CDs
 285
 286         my $devdir = "/sys/block/$dev/device";
 287         return if ! -d $devdir;
 288
 289         my $size = file_read_firstline("/sys/block/$dev/size");
 290         return if !$size;
 291
 292         $size = $size * 512;
 293
 294         my $info = `udevadm info --path /sys/block/$dev --query all`;
 295         return if !$info;
 296
 297         return if $info !~ m/^E: DEVTYPE=disk$/m;
 298         return if $info =~ m/^E: ID_CDROM/m;
 299
 300         my $serial = 'unknown';
 301         if ($info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m) {
 302             $serial = $1;
 303         }
 304
 305         my $gpt = 0;
 306         if ($info =~ m/^E: ID_PART_TABLE_TYPE=gpt$/m) {
 307             $gpt = 1;
 308         }
 309
 310         # detect SSD (fixme - currently only works for ATA disks)
 311         my $rpm = 7200; # default guess
 312         if ($info =~ m/^E: ID_ATA_ROTATION_RATE_RPM=(\d+)$/m) {
 313             $rpm = $1;
 314         }
 315
 316         my $vendor = file_read_firstline("$devdir/vendor") || 'unknown';
 317         my $model = file_read_firstline("$devdir/model") || 'unknown';
 318
 319         my $used;
 320
 321         $used = 'LVM' if !&$dir_is_epmty("/sys/block/$dev/holders");
 322
 323         $used = 'mounted' if &$dev_is_mounted("/dev/$dev");
 324
 325         $disklist->{$dev} = {
 326             vendor => $vendor,
 327             model => $model,
 328             size => $size,
 329             serial => $serial,
 330             gpt => $gpt,
 331             rmp => $rpm,
 332         };
 333
 334         my $osdid = -1;
 335
 336         my $journal_count = 0;
 337
 338         my $found_partitions;
 339         my $found_lvm;
 340         my $found_mountpoints;
 341         dir_glob_foreach("/sys/block/$dev", "$dev.+", sub {
 342             my ($part) = @_;
 343
 344             $found_partitions = 1;
 345
 346             if (my $mp = &$dev_is_mounted("/dev/$part")) {
 347                 $found_mountpoints = 1;
 348                 if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) {
 349                     $osdid = $1;
 350                 }
 351             }
 352             if (!&$dir_is_epmty("/sys/block/$dev/$part/holders"))  {
 353                 $found_lvm = 1;
 354             }
 355             $journal_count++ if $journalhash->{"/dev/$part"};
 356         });
 357
 358         $used = 'mounted' if $found_mountpoints && !$used;
 359         $used = 'LVM' if $found_lvm && !$used;
 360         $used = 'partitions' if $found_partitions && !$used;
 361
 362         $disklist->{$dev}->{used} = $used if $used;
 363         $disklist->{$dev}->{osdid} = $osdid;
 364         $disklist->{$dev}->{journals} = $journal_count;
 365     });
 366
 367     return $disklist;
 368 }
 369
 370 my $lookup_diskinfo = sub {
 371     my ($disklist, $disk) = @_;
 372
 373     my $real_dev = abs_path($disk);
 374     $real_dev =~ s|/dev/||;
 375     my $diskinfo = $disklist->{$real_dev};
 376
 377     die "disk '$disk' not found in disk list\n" if !$diskinfo;
 378
 379     return wantarray ? ($diskinfo, $real_dev) : $diskinfo;
 380 };
 381
 382
 383 my $count_journal_disks = sub {
 384     my ($disklist, $disk) = @_;
 385
 386     my $count = 0;
 387
 388     my ($diskinfo, $real_dev) = &$lookup_diskinfo($disklist, $disk);
 389     die "journal disk '$disk' does not contain a GUID partition table\n"
 390         if !$diskinfo->{gpt};
 391
 392     $count = $diskinfo->{journals} if $diskinfo->{journals};
 393
 394     return $count;
 395 };
 396
 397 __PACKAGE__->register_method ({
 398     name => 'index',
 399     path => '',
 400     method => 'GET',
 401     description => "Directory index.",
 402     permissions => { user => 'all' },
 403     parameters => {
 404         additionalProperties => 0,
 405         properties => {
 406             node => get_standard_option('pve-node'),
 407         },
 408     },
 409     returns => {
 410         type => 'array',
 411         items => {
 412             type => "object",
 413             properties => {},
 414         },
 415         links => [ { rel => 'child', href => "{name}" } ],
 416     },
 417     code => sub {
 418         my ($param) = @_;
 419
 420         my $result = [
 421             { name => 'init' },
 422             { name => 'mon' },
 423             { name => 'osd' },
 424             { name => 'pools' },
 425             { name => 'stop' },
 426             { name => 'start' },
 427             { name => 'status' },
 428             { name => 'crush' },
 429             { name => 'config' },
 430             { name => 'log' },
 431             { name => 'disks' },
 432         ];
 433
 434         return $result;
 435     }});
 436
 437 __PACKAGE__->register_method ({
 438     name => 'disks',
 439     path => 'disks',
 440     method => 'GET',
 441     description => "List local disks.",
 442     proxyto => 'node',
 443     protected => 1,
 444     parameters => {
 445         additionalProperties => 0,
 446         properties => {
 447             node => get_standard_option('pve-node'),
 448             type => {
 449                 description => "Only list specific types of disks.",
 450                 type => 'string',
 451                 enum => ['unused', 'journal_disks'],
 452                 optional => 1,
 453             },
 454         },
 455     },
 456     returns => {
 457         type => 'array',
 458         items => {
 459             type => "object",
 460             properties => {
 461                 dev => { type => 'string' },
 462                 used => { type => 'string', optional => 1 },
 463                 gpt => { type => 'boolean' },
 464                 size => { type => 'integer' },
 465                 osdid => { type => 'integer' },
 466                 vendor =>  { type => 'string', optional => 1 },
 467                 model =>  { type => 'string', optional => 1 },
 468                 serial =>  { type => 'string', optional => 1 },
 469             },
 470         },
 471         # links => [ { rel => 'child', href => "{}" } ],
 472     },
 473     code => sub {
 474         my ($param) = @_;
 475
 476         &$check_ceph_inited();
 477
 478         my $disks = list_disks();
 479
 480         my $res = [];
 481         foreach my $dev (keys %$disks) {
 482             my $d = $disks->{$dev};
 483             if ($param->{type}) {
 484                 if ($param->{type} eq 'journal_disks') {
 485                     next if $d->{osdid} >= 0;
 486                     next if !$d->{gpt};
 487                 } elsif ($param->{type} eq 'unused') {
 488                     next if $d->{used};
 489                 } else {
 490                     die "internal error"; # should not happen
 491                 }
 492             }
 493
 494             $d->{dev} = "/dev/$dev";
 495             push @$res, $d;
 496         }
 497
 498         return $res;
 499     }});
 500
 501 __PACKAGE__->register_method ({
 502     name => 'config',
 503     path => 'config',
 504     method => 'GET',
 505     description => "Get Ceph configuration.",
 506     parameters => {
 507         additionalProperties => 0,
 508         properties => {
 509             node => get_standard_option('pve-node'),
 510         },
 511     },
 512     returns => { type => 'string' },
 513     code => sub {
 514         my ($param) = @_;
 515
 516         &$check_ceph_inited();
 517
 518         return PVE::Tools::file_get_contents($pve_ceph_cfgpath);
 519
 520     }});
 521
 522 __PACKAGE__->register_method ({
 523     name => 'listmon',
 524     path => 'mon',
 525     method => 'GET',
 526     description => "Get Ceph monitor list.",
 527     proxyto => 'node',
 528     protected => 1,
 529     parameters => {
 530         additionalProperties => 0,
 531         properties => {
 532             node => get_standard_option('pve-node'),
 533         },
 534     },
 535     returns => {
 536         type => 'array',
 537         items => {
 538             type => "object",
 539             properties => {
 540                 name => { type => 'string' },
 541                 addr => { type => 'string' },
 542             },
 543         },
 544         links => [ { rel => 'child', href => "{name}" } ],
 545     },
 546     code => sub {
 547         my ($param) = @_;
 548
 549         &$check_ceph_inited();
 550
 551         my $res = [];
 552
 553         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 554
 555         my $monhash = {};
 556         foreach my $section (keys %$cfg) {
 557             my $d = $cfg->{$section};
 558             if ($section =~ m/^mon\.(\S+)$/) {
 559                 my $monid = $1;
 560                 if ($d->{'mon addr'} && $d->{'host'}) {
 561                     $monhash->{$monid} = {
 562                         addr => $d->{'mon addr'},
 563                         host => $d->{'host'},
 564                         name => $monid,
 565                     }
 566                 }
 567             }
 568         }
 569
 570         eval {
 571             my $monstat = ceph_mon_status();
 572             my $mons = $monstat->{monmap}->{mons};
 573             foreach my $d (@$mons) {
 574                 next if !defined($d->{name});
 575                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 576                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 577                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 578                     $monhash->{$d->{name}}->{quorum} = 1;
 579                 }
 580             }
 581         };
 582         warn $@ if $@;
 583
 584         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 585     }});
 586
 587 __PACKAGE__->register_method ({
 588     name => 'init',
 589     path => 'init',
 590     method => 'POST',
 591     description => "Create initial ceph default configuration and setup symlinks.",
 592     proxyto => 'node',
 593     protected => 1,
 594     parameters => {
 595         additionalProperties => 0,
 596         properties => {
 597             node => get_standard_option('pve-node'),
 598             network => {
 599                 description => "Use specific network for all ceph related traffic",
 600                 type => 'string', format => 'CIDR',
 601                 optional => 1,
 602                 maxLength => 128,
 603             },
 604             size => {
 605                 description => 'Number of replicas per object',
 606                 type => 'integer',
 607                 default => 2,
 608                 optional => 1,
 609                 minimum => 1,
 610                 maximum => 3,
 611             },
 612             pg_bits => {
 613                 description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)",
 614                 type => 'integer',
 615                 default => 6,
 616                 optional => 1,
 617                 minimum => 6,
 618                 maximum => 14,
 619             },
 620         },
 621     },
 622     returns => { type => 'null' },
 623     code => sub {
 624         my ($param) = @_;
 625
 626         &$check_ceph_installed();
 627
 628         # simply load old config if it already exists
 629         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 630
 631         if (!$cfg->{global}) {
 632
 633             my $fsid;
 634             my $uuid;
 635
 636             UUID::generate($uuid);
 637             UUID::unparse($uuid, $fsid);
 638
 639             $cfg->{global} = {
 640                 'fsid' => $fsid,
 641                 'auth supported' => 'cephx',
 642                 'auth cluster required' => 'cephx',
 643                 'auth service required' => 'cephx',
 644                 'auth client required' => 'cephx',
 645                 'filestore xattr use omap' => 'true',
 646                 'osd journal size' => $pve_osd_default_journal_size,
 647                 'osd pool default min size' => 1,
 648             };
 649
 650             # this does not work for default pools
 651             #'osd pool default pg num' => $pg_num,
 652             #'osd pool default pgp num' => $pg_num,
 653         }
 654
 655         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 656         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 657
 658         $cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size};
 659
 660         if ($param->{pg_bits}) {
 661             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 662             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 663         }
 664
 665         if ($param->{network}) {
 666             $cfg->{global}->{'public network'} = $param->{network};
 667             $cfg->{global}->{'cluster network'} = $param->{network};
 668         }
 669
 670         &$write_ceph_config($cfg);
 671
 672         &$setup_pve_symlinks();
 673
 674         return undef;
 675     }});
 676
 677 my $find_node_ip = sub {
 678     my ($cidr) = @_;
 679
 680     my $config = PVE::INotify::read_file('interfaces');
 681
 682     my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n";
 683
 684     foreach my $iface (keys %$config) {
 685         my $d = $config->{$iface};
 686         next if !$d->{address};
 687         my $a = Net::IP->new($d->{address});
 688         next if !$a;
 689         return $d->{address} if $net->overlaps($a);
 690     }
 691
 692     die "unable to find local address within network '$cidr'\n";
 693 };
 694
 695 __PACKAGE__->register_method ({
 696     name => 'createmon',
 697     path => 'mon',
 698     method => 'POST',
 699     description => "Create Ceph Monitor",
 700     proxyto => 'node',
 701     protected => 1,
 702     parameters => {
 703         additionalProperties => 0,
 704         properties => {
 705             node => get_standard_option('pve-node'),
 706         },
 707     },
 708     returns => { type => 'string' },
 709     code => sub {
 710         my ($param) = @_;
 711
 712         &$check_ceph_inited();
 713
 714         &$setup_pve_symlinks();
 715
 716         my $rpcenv = PVE::RPCEnvironment::get();
 717
 718         my $authuser = $rpcenv->get_user();
 719
 720         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 721
 722         my $moncount = 0;
 723
 724         my $monaddrhash = {};
 725
 726         foreach my $section (keys %$cfg) {
 727             next if $section eq 'global';
 728             my $d = $cfg->{$section};
 729             if ($section =~ m/^mon\./) {
 730                 $moncount++;
 731                 if ($d->{'mon addr'}) {
 732                     $monaddrhash->{$d->{'mon addr'}} = $section;
 733                 }
 734             }
 735         }
 736
 737         my $monid;
 738         for (my $i = 0; $i < 7; $i++) {
 739             if (!$cfg->{"mon.$i"}) {
 740                 $monid = $i;
 741                 last;
 742             }
 743         }
 744         die "unable to find usable monitor id\n" if !defined($monid);
 745
 746         my $monsection = "mon.$monid";
 747         my $ip;
 748         if (my $pubnet = $cfg->{global}->{'public network'}) {
 749             $ip = &$find_node_ip($pubnet);
 750         } else {
 751             $ip = PVE::Cluster::remote_node_ip($param->{node});
 752         }
 753
 754         my $monaddr = "$ip:6789";
 755         my $monname = $param->{node};
 756
 757         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
 758         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
 759             if $monaddrhash->{$monaddr};
 760
 761         my $worker = sub  {
 762             my $upid = shift;
 763
 764             if (! -f $pve_ckeyring_path) {
 765                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
 766                             "--gen-key -n client.admin");
 767             }
 768
 769             if (! -f $pve_mon_key_path) {
 770                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
 771                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
 772                             "--cap mds 'allow' " .
 773                             "--cap osd 'allow *' " .
 774                             "--cap mon 'allow *'");
 775                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
 776                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
 777             }
 778
 779             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 780             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
 781
 782             my $monmap = "/tmp/monmap";
 783
 784             eval {
 785                 mkdir $mondir;
 786
 787                 if ($moncount > 0) {
 788                     my $monstat = ceph_mon_status(); # online test
 789                     &$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]);
 790                 } else {
 791                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
 792                 }
 793
 794                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
 795             };
 796             my $err = $@;
 797             unlink $monmap;
 798             if ($err) {
 799                 File::Path::remove_tree($mondir);
 800                 die $err;
 801             }
 802
 803             $cfg->{$monsection} = {
 804                 'host' => $monname,
 805                 'mon addr' => $monaddr,
 806             };
 807
 808             &$write_ceph_config($cfg);
 809
 810             &$ceph_service_cmd('start', $monsection);
 811         };
 812
 813         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
 814     }});
 815
 816 __PACKAGE__->register_method ({
 817     name => 'destroymon',
 818     path => 'mon/{monid}',
 819     method => 'DELETE',
 820     description => "Destroy Ceph monitor.",
 821     proxyto => 'node',
 822     protected => 1,
 823     parameters => {
 824         additionalProperties => 0,
 825         properties => {
 826             node => get_standard_option('pve-node'),
 827             monid => {
 828                 description => 'Monitor ID',
 829                 type => 'integer',
 830             },
 831         },
 832     },
 833     returns => { type => 'string' },
 834     code => sub {
 835         my ($param) = @_;
 836
 837         my $rpcenv = PVE::RPCEnvironment::get();
 838
 839         my $authuser = $rpcenv->get_user();
 840
 841         &$check_ceph_inited();
 842
 843         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 844
 845         my $monid = $param->{monid};
 846         my $monsection = "mon.$monid";
 847
 848         my $monstat = ceph_mon_status();
 849         my $monlist = $monstat->{monmap}->{mons};
 850
 851         die "no such monitor id '$monid'\n"
 852             if !defined($cfg->{$monsection});
 853
 854
 855         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 856         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
 857
 858         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
 859
 860         my $worker = sub {
 861             my $upid = shift;
 862
 863             &$run_ceph_cmd(['mon', 'remove', $monid]);
 864
 865             eval { &$ceph_service_cmd('stop', $monsection); };
 866             warn $@ if $@;
 867
 868             delete $cfg->{$monsection};
 869             &$write_ceph_config($cfg);
 870             File::Path::remove_tree($mondir);
 871         };
 872
 873         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
 874     }});
 875
 876 __PACKAGE__->register_method ({
 877     name => 'stop',
 878     path => 'stop',
 879     method => 'POST',
 880     description => "Stop ceph services.",
 881     proxyto => 'node',
 882     protected => 1,
 883     parameters => {
 884         additionalProperties => 0,
 885         properties => {
 886             node => get_standard_option('pve-node'),
 887             service => {
 888                 description => 'Ceph service name.',
 889                 type => 'string',
 890                 optional => 1,
 891                 pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
 892             },
 893         },
 894     },
 895     returns => { type => 'string' },
 896     code => sub {
 897         my ($param) = @_;
 898
 899         my $rpcenv = PVE::RPCEnvironment::get();
 900
 901         my $authuser = $rpcenv->get_user();
 902
 903         &$check_ceph_inited();
 904
 905         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 906         scalar(keys %$cfg) || die "no configuration\n";
 907
 908         my $worker = sub {
 909             my $upid = shift;
 910
 911             my $cmd = ['stop'];
 912             if ($param->{service}) {
 913                 push @$cmd, $param->{service};
 914             }
 915
 916             &$ceph_service_cmd(@$cmd);
 917         };
 918
 919         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 920                                     $authuser, $worker);
 921     }});
 922
 923 __PACKAGE__->register_method ({
 924     name => 'start',
 925     path => 'start',
 926     method => 'POST',
 927     description => "Start ceph services.",
 928     proxyto => 'node',
 929     protected => 1,
 930     parameters => {
 931         additionalProperties => 0,
 932         properties => {
 933             node => get_standard_option('pve-node'),
 934             service => {
 935                 description => 'Ceph service name.',
 936                 type => 'string',
 937                 optional => 1,
 938                 pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
 939             },
 940         },
 941     },
 942     returns => { type => 'string' },
 943     code => sub {
 944         my ($param) = @_;
 945
 946         my $rpcenv = PVE::RPCEnvironment::get();
 947
 948         my $authuser = $rpcenv->get_user();
 949
 950         &$check_ceph_inited();
 951
 952         my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
 953         scalar(keys %$cfg) || die "no configuration\n";
 954
 955         my $worker = sub {
 956             my $upid = shift;
 957
 958             my $cmd = ['start'];
 959             if ($param->{service}) {
 960                 push @$cmd, $param->{service};
 961             }
 962
 963             &$ceph_service_cmd(@$cmd);
 964         };
 965
 966         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 967                                     $authuser, $worker);
 968     }});
 969
 970 __PACKAGE__->register_method ({
 971     name => 'status',
 972     path => 'status',
 973     method => 'GET',
 974     description => "Get ceph status.",
 975     proxyto => 'node',
 976     protected => 1,
 977     parameters => {
 978         additionalProperties => 0,
 979         properties => {
 980             node => get_standard_option('pve-node'),
 981         },
 982     },
 983     returns => { type => 'object' },
 984     code => sub {
 985         my ($param) = @_;
 986
 987         &$check_ceph_enabled();
 988
 989         return &$run_ceph_cmd_json(['status'], quiet => 1);
 990     }});
 991
 992 __PACKAGE__->register_method ({
 993     name => 'lspools',
 994     path => 'pools',
 995     method => 'GET',
 996     description => "List all pools.",
 997     proxyto => 'node',
 998     protected => 1,
 999     parameters => {
1000         additionalProperties => 0,
1001         properties => {
1002             node => get_standard_option('pve-node'),
1003         },
1004     },
1005     returns => {
1006         type => 'array',
1007         items => {
1008             type => "object",
1009             properties => {
1010                 pool => { type => 'integer' },
1011                 pool_name => { type => 'string' },
1012                 size => { type => 'integer' },
1013             },
1014         },
1015         links => [ { rel => 'child', href => "{pool_name}" } ],
1016     },
1017     code => sub {
1018         my ($param) = @_;
1019
1020         &$check_ceph_inited();
1021
1022         my $res = &$run_ceph_cmd_json(['osd', 'dump'], quiet => 1);
1023
1024         my $data = [];
1025         foreach my $e (@{$res->{pools}}) {
1026             my $d = {};
1027             foreach my $attr (qw(pool pool_name size min_size pg_num crush_ruleset)) {
1028                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1029             }
1030             push @$data, $d;
1031         }
1032
1033         return $data;
1034     }});
1035
1036 __PACKAGE__->register_method ({
1037     name => 'createpool',
1038     path => 'pools',
1039     method => 'POST',
1040     description => "Create POOL",
1041     proxyto => 'node',
1042     protected => 1,
1043     parameters => {
1044         additionalProperties => 0,
1045         properties => {
1046             node => get_standard_option('pve-node'),
1047             name => {
1048                 description => "The name of the pool. It must be unique.",
1049                 type => 'string',
1050             },
1051             size => {
1052                 description => 'Number of replicas per object',
1053                 type => 'integer',
1054                 default => 2,
1055                 optional => 1,
1056                 minimum => 1,
1057                 maximum => 3,
1058             },
1059             min_size => {
1060                 description => 'Minimum number of replicas per object',
1061                 type => 'integer',
1062                 default => 1,
1063                 optional => 1,
1064                 minimum => 1,
1065                 maximum => 3,
1066             },
1067             pg_num => {
1068                 description => "Number of placement groups.",
1069                 type => 'integer',
1070                 default => 64,
1071                 optional => 1,
1072                 minimum => 8,
1073                 maximum => 32768,
1074             },
1075             crush_ruleset => {
1076                 description => "The ruleset to use for mapping object placement in the cluster.",
1077                 type => 'integer',
1078                 minimum => 0,
1079                 maximum => 32768,
1080                 default => 0,
1081                 optional => 1,
1082             },
1083         },
1084     },
1085     returns => { type => 'null' },
1086     code => sub {
1087         my ($param) = @_;
1088
1089         &$check_ceph_inited();
1090
1091         die "not fully configured - missing '$pve_ckeyring_path'\n"
1092             if ! -f $pve_ckeyring_path;
1093
1094         my $pg_num = $param->{pg_num} || 64;
1095         my $size = $param->{size} || 2;
1096         my $min_size = $param->{min_size} || 1;
1097
1098         &$run_ceph_cmd(['osd', 'pool', 'create', $param->{name}, $pg_num]);
1099
1100         &$run_ceph_cmd(['osd', 'pool', 'set', $param->{name}, 'min_size', $min_size]);
1101
1102         &$run_ceph_cmd(['osd', 'pool', 'set', $param->{name}, 'size', $size]);
1103
1104         if (defined($param->{crush_ruleset})) {
1105             &$run_ceph_cmd(['osd', 'pool', 'set', $param->{name}, 'crush_ruleset', $param->{crush_ruleset}]);
1106         }
1107
1108         return undef;
1109     }});
1110
1111 __PACKAGE__->register_method ({
1112     name => 'destroypool',
1113     path => 'pools/{name}',
1114     method => 'DELETE',
1115     description => "Destroy pool",
1116     proxyto => 'node',
1117     protected => 1,
1118     parameters => {
1119         additionalProperties => 0,
1120         properties => {
1121             node => get_standard_option('pve-node'),
1122             name => {
1123                 description => "The name of the pool. It must be unique.",
1124                 type => 'string',
1125             },
1126         },
1127     },
1128     returns => { type => 'null' },
1129     code => sub {
1130         my ($param) = @_;
1131
1132         &$check_ceph_inited();
1133
1134         &$run_ceph_cmd(['osd', 'pool', 'delete', $param->{name}, $param->{name}, '--yes-i-really-really-mean-it']);
1135
1136         return undef;
1137     }});
1138
1139 __PACKAGE__->register_method ({
1140     name => 'listosd',
1141     path => 'osd',
1142     method => 'GET',
1143     description => "Get Ceph osd list/tree.",
1144     proxyto => 'node',
1145     protected => 1,
1146     parameters => {
1147         additionalProperties => 0,
1148         properties => {
1149             node => get_standard_option('pve-node'),
1150         },
1151     },
1152     returns => {
1153         type => "object",
1154     },
1155     code => sub {
1156         my ($param) = @_;
1157
1158         &$check_ceph_inited();
1159
1160         my $res = &$run_ceph_cmd_json(['osd', 'tree'], quiet => 1);
1161
1162         die "no tree nodes found\n" if !($res && $res->{nodes});
1163
1164         my $nodes = {};
1165         my $newnodes = {};
1166         foreach my $e (@{$res->{nodes}}) {
1167             $nodes->{$e->{id}} = $e;
1168
1169             my $new = {
1170                 id => $e->{id},
1171                 name => $e->{name},
1172                 type => $e->{type}
1173             };
1174
1175             foreach my $opt (qw(status crush_weight reweight)) {
1176                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
1177             }
1178
1179             $newnodes->{$e->{id}} = $new;
1180         }
1181
1182         foreach my $e (@{$res->{nodes}}) {
1183             my $new = $newnodes->{$e->{id}};
1184             if ($e->{children} && scalar(@{$e->{children}})) {
1185                 $new->{children} = [];
1186                 $new->{leaf} = 0;
1187                 foreach my $cid (@{$e->{children}}) {
1188                     $nodes->{$cid}->{parent} = $e->{id};
1189                     if ($nodes->{$cid}->{type} eq 'osd' &&
1190                         $e->{type} eq 'host') {
1191                         $newnodes->{$cid}->{host} = $e->{name};
1192                     }
1193                     push @{$new->{children}}, $newnodes->{$cid};
1194                 }
1195             } else {
1196                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
1197             }
1198         }
1199
1200         my $rootnode;
1201         foreach my $e (@{$res->{nodes}}) {
1202             if (!$nodes->{$e->{id}}->{parent}) {
1203                 $rootnode = $newnodes->{$e->{id}};
1204                 last;
1205             }
1206         }
1207
1208         die "no root node\n" if !$rootnode;
1209
1210         my $data = { root => $rootnode };
1211
1212         return $data;
1213     }});
1214
1215 __PACKAGE__->register_method ({
1216     name => 'createosd',
1217     path => 'osd',
1218     method => 'POST',
1219     description => "Create OSD",
1220     proxyto => 'node',
1221     protected => 1,
1222     parameters => {
1223         additionalProperties => 0,
1224         properties => {
1225             node => get_standard_option('pve-node'),
1226             dev => {
1227                 description => "Block device name.",
1228                 type => 'string',
1229             },
1230             journal_dev => {
1231                 description => "Block device name for journal.",
1232                 optional => 1,
1233                 type => 'string',
1234             },
1235             fstype => {
1236                 description => "File system type.",
1237                 type => 'string',
1238                 enum => ['xfs', 'ext4', 'btrfs'],
1239                 default => 'xfs',
1240                 optional => 1,
1241             },
1242         },
1243     },
1244     returns => { type => 'string' },
1245     code => sub {
1246         my ($param) = @_;
1247
1248         my $rpcenv = PVE::RPCEnvironment::get();
1249
1250         my $authuser = $rpcenv->get_user();
1251
1252         &$check_ceph_inited();
1253
1254         &$setup_pve_symlinks();
1255
1256         my $journal_dev;
1257
1258         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
1259             -b  $param->{journal_dev} || die "no such block device '$param->{journal_dev}'\n";
1260             $journal_dev = $param->{journal_dev};
1261         }
1262
1263         -b  $param->{dev} || die "no such block device '$param->{dev}'\n";
1264
1265         my $disklist = list_disks();
1266
1267         my $devname = $param->{dev};
1268         $devname =~ s|/dev/||;
1269
1270         my $diskinfo = $disklist->{$devname};
1271         die "unable to get device info for '$devname'\n"
1272             if !$diskinfo;
1273
1274         die "device '$param->{dev}' is in use\n"
1275             if $diskinfo->{used};
1276
1277         my $monstat = ceph_mon_status(1);
1278         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
1279         my $fsid = $monstat->{monmap}->{fsid};
1280
1281         if (! -f $ceph_bootstrap_osd_keyring) {
1282             &$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]);
1283         };
1284
1285         my $worker = sub {
1286             my $upid = shift;
1287
1288             my $fstype = $param->{fstype} || 'xfs';
1289
1290             print "create OSD on $param->{dev} ($fstype)\n";
1291
1292             my $cmd = ['ceph-disk', 'prepare', '--zap-disk', '--fs-type', $fstype,
1293                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
1294
1295             if ($journal_dev) {
1296                 print "using device '$journal_dev' for journal\n";
1297                 push @$cmd, '--journal-dev', $param->{dev}, $journal_dev;
1298             } else {
1299                 push @$cmd, $param->{dev};
1300             }
1301
1302             run_command($cmd);
1303         };
1304
1305         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
1306     }});
1307
1308 __PACKAGE__->register_method ({
1309     name => 'destroyosd',
1310     path => 'osd/{osdid}',
1311     method => 'DELETE',
1312     description => "Destroy OSD",
1313     proxyto => 'node',
1314     protected => 1,
1315     parameters => {
1316         additionalProperties => 0,
1317         properties => {
1318             node => get_standard_option('pve-node'),
1319             osdid => {
1320                 description => 'OSD ID',
1321                 type => 'integer',
1322             },
1323             cleanup => {
1324                 description => "If set, we remove partition table entries.",
1325                 type => 'boolean',
1326                 optional => 1,
1327                 default => 0,
1328             },
1329         },
1330     },
1331     returns => { type => 'string' },
1332     code => sub {
1333         my ($param) = @_;
1334
1335         my $rpcenv = PVE::RPCEnvironment::get();
1336
1337         my $authuser = $rpcenv->get_user();
1338
1339         &$check_ceph_inited();
1340
1341         my $osdid = $param->{osdid};
1342
1343         # fixme: not 100% sure what we should do here
1344
1345         my $stat = &$ceph_osd_status();
1346
1347         my $osdlist = $stat->{osds} || [];
1348
1349         my $osdstat;
1350         foreach my $d (@$osdlist) {
1351             if ($d->{osd} == $osdid) {
1352                 $osdstat = $d;
1353                 last;
1354             }
1355         }
1356         die "no such OSD '$osdid'\n" if !$osdstat;
1357
1358         die "osd is in use (in == 1)\n" if $osdstat->{in};
1359         #&$run_ceph_cmd(['osd', 'out', $osdid]);
1360
1361         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
1362
1363         my $osdsection = "osd.$osdid";
1364
1365         my $worker = sub {
1366             my $upid = shift;
1367
1368             print "destroy OSD $osdsection\n";
1369
1370             eval { &$ceph_service_cmd('stop', $osdsection); };
1371             warn $@ if $@;
1372
1373             print "Remove $osdsection from the CRUSH map\n";
1374             &$run_ceph_cmd(['osd', 'crush', 'remove', $osdsection]);
1375
1376             print "Remove the $osdsection authentication key.\n";
1377             &$run_ceph_cmd(['auth', 'del', $osdsection]);
1378
1379             print "Remove OSD $osdsection\n";
1380             &$run_ceph_cmd(['osd', 'rm', $osdid]);
1381
1382             # try to unmount from standard mount point
1383             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
1384
1385             my $remove_partition = sub {
1386                 my ($disklist, $part) = @_;
1387
1388                 return if !$part || (! -b $part );
1389
1390                 foreach my $real_dev (keys %$disklist) {
1391                     my $diskinfo = $disklist->{$real_dev};
1392                     next if !$diskinfo->{gpt};
1393                     if ($part =~ m|^/dev/${real_dev}(\d+)$|) {
1394                         my $partnum = $1;
1395                         print "remove partition $part (disk '/dev/${real_dev}', partnum $partnum)\n";
1396                         eval { run_command(['/sbin/sgdisk', '-d', $partnum, "/dev/${real_dev}"]); };
1397                         warn $@ if $@;
1398                         last;
1399                     }
1400                 }
1401             };
1402
1403             my $journal_part;
1404             my $data_part;
1405
1406             if ($param->{cleanup}) {
1407                 my $jpath = "$mountpoint/journal";
1408                 $journal_part = abs_path($jpath);
1409
1410                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
1411                     while (defined(my $line = <$fd>)) {
1412                         my ($dev, $path, $fstype) = split(/\s+/, $line);
1413                         next if !($dev && $path && $fstype);
1414                         next if $dev !~ m|^/dev/|;
1415                         if ($path eq $mountpoint) {
1416                             $data_part = abs_path($dev);
1417                             last;
1418                         }
1419                     }
1420                     close($fd);
1421                 }
1422             }
1423
1424             print "Unmount OSD $osdsection from  $mountpoint\n";
1425             eval { run_command(['umount', $mountpoint]); };
1426             if (my $err = $@) {
1427                 warn $err;
1428             } elsif ($param->{cleanup}) {
1429                 my $disklist = list_disks();
1430                 &$remove_partition($disklist, $journal_part);
1431                 &$remove_partition($disklist, $data_part);
1432             }
1433         };
1434
1435         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
1436     }});
1437
1438 __PACKAGE__->register_method ({
1439     name => 'crush',
1440     path => 'crush',
1441     method => 'GET',
1442     description => "Get OSD crush map",
1443     proxyto => 'node',
1444     protected => 1,
1445     parameters => {
1446         additionalProperties => 0,
1447         properties => {
1448             node => get_standard_option('pve-node'),
1449         },
1450     },
1451     returns => { type => 'string' },
1452     code => sub {
1453         my ($param) = @_;
1454
1455         &$check_ceph_inited();
1456
1457         # this produces JSON (difficult to read for the user)
1458         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1459
1460         my $txt = '';
1461
1462         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1463         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1464
1465         eval {
1466             &$run_ceph_cmd(['osd', 'getcrushmap', '-o', $mapfile]);
1467             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1468             $txt = PVE::Tools::file_get_contents($mapdata);
1469         };
1470         my $err = $@;
1471
1472         unlink $mapfile;
1473         unlink $mapdata;
1474
1475         die $err if $err;
1476
1477         return $txt;
1478     }});
1479
1480 __PACKAGE__->register_method({
1481     name => 'log',
1482     path => 'log',
1483     method => 'GET',
1484     description => "Read ceph log",
1485     proxyto => 'node',
1486     permissions => {
1487         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1488     },
1489     protected => 1,
1490     parameters => {
1491         additionalProperties => 0,
1492         properties => {
1493             node => get_standard_option('pve-node'),
1494             start => {
1495                 type => 'integer',
1496                 minimum => 0,
1497                 optional => 1,
1498             },
1499             limit => {
1500                 type => 'integer',
1501                 minimum => 0,
1502                 optional => 1,
1503             },
1504         },
1505     },
1506     returns => {
1507         type => 'array',
1508         items => {
1509             type => "object",
1510             properties => {
1511                 n => {
1512                   description=>  "Line number",
1513                   type=> 'integer',
1514                 },
1515                 t => {
1516                   description=>  "Line text",
1517                   type => 'string',
1518                 }
1519             }
1520         }
1521     },
1522     code => sub {
1523         my ($param) = @_;
1524
1525         my $rpcenv = PVE::RPCEnvironment::get();
1526         my $user = $rpcenv->get_user();
1527         my $node = $param->{node};
1528
1529         my $logfile = "/var/log/ceph/ceph.log";
1530         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1531
1532         $rpcenv->set_result_attrib('total', $count);
1533
1534         return $lines;
1535     }});
1536
1537