PVE/RADOS.pm

   1 package PVE::RADOS;
   2
   3 use 5.014002;
   4
   5 use strict;
   6 use warnings;
   7
   8 use Carp;
   9 use JSON;
  10 use POSIX;
  11 use Socket;
  12
  13 use PVE::Tools;
  14 use PVE::INotify;
  15 use PVE::RPCEnvironment;
  16
  17 require Exporter;
  18
  19 my $rados_default_timeout = 5;
  20 my $ceph_default_conf = '/etc/pve/ceph.conf';
  21 my $ceph_default_user = 'admin';
  22
  23
  24 our $VERSION = '1.0';
  25
  26 require XSLoader;
  27 XSLoader::load('PVE::RADOS', $VERSION);
  28
  29 my $writedata = sub {
  30     my ($fh, $cmd, $data) = @_;
  31
  32     local $SIG{PIPE} = 'IGNORE';
  33
  34     my $bin = pack "a L/a*", $cmd, $data || '';
  35     my $res = syswrite $fh, $bin;
  36
  37     die "write data failed - $!\n" if !defined($res);
  38 };
  39
  40 my $readdata = sub {
  41     my ($fh, $allow_eof) = @_;
  42
  43     my $head = '';
  44
  45     local $SIG{PIPE} = 'IGNORE';
  46
  47     while (length($head) < 5) {
  48         last if !sysread $fh, $head, 5 - length($head), length($head);
  49     }
  50     return undef if $allow_eof && length($head) == 0;
  51
  52     die "partial read\n" if length($head) < 5;
  53
  54     my ($cmd, $len) = unpack "a L", $head;
  55
  56     my $data = '';
  57     while (length($data) < $len) {
  58         last if !sysread $fh, $data, $len - length($data), length($data);
  59     }
  60     die "partial data read\n" if length($data) < $len;
  61
  62     return wantarray ? ($cmd, $data) : $data;
  63 };
  64
  65 my $kill_worker = sub {
  66     my ($self) = @_;
  67
  68     return if !$self->{cpid};
  69     return if  $self->{__already_killed};
  70
  71     $self->{__already_killed} = 1;
  72
  73     close($self->{child}) if defined($self->{child});
  74
  75     # only kill if we created the process
  76     return if $self->{pid} != $$;
  77
  78     kill(9, $self->{cpid});
  79     waitpid($self->{cpid}, 0);
  80 };
  81
  82 my $sendcmd = sub {
  83     my ($self, $cmd, $data, $expect_tag) = @_;
  84
  85     $expect_tag = '>' if !$expect_tag;
  86
  87     die "detected forked connection\n" if $self->{pid} != $$;
  88
  89     my ($restag, $raw);
  90     my $code = sub {
  91         &$writedata($self->{child}, $cmd, $data) if $expect_tag ne 'S';
  92         ($restag, $raw) = &$readdata($self->{child});
  93     };
  94     eval { PVE::Tools::run_with_timeout($self->{timeout}, $code); };
  95     if (my $err = $@) {
  96         &$kill_worker($self);
  97         die $err;
  98     }
  99     if ($restag eq 'E') {
 100         die $raw if $raw;
 101         die "unknown error\n";
 102     }
 103
 104     die "got unexpected result\n" if $restag ne $expect_tag;
 105
 106     return $raw;
 107 };
 108
 109 sub pve_rados_work {
 110     my ($self, $parent, $timeout, %params) = @_;
 111
 112     my $conn;
 113     eval {
 114         my $ceph_user = delete $params{userid} || $ceph_default_user;
 115         $conn = pve_rados_create($ceph_user) ||
 116             die "unable to create RADOS object\n";
 117
 118         if (defined($params{ceph_conf}) && (!-e $params{ceph_conf})) {
 119             die "Supplied ceph config doesn't exist, $params{ceph_conf}\n";
 120         }
 121
 122         my $ceph_conf = delete $params{ceph_conf} || $ceph_default_conf;
 123
 124         if (-e $ceph_conf) {
 125             pve_rados_conf_read_file($conn, $ceph_conf);
 126         }
 127
 128         pve_rados_conf_set($conn, 'client_mount_timeout', $timeout);
 129
 130         foreach my $k (keys %params) {
 131             pve_rados_conf_set($conn, $k, $params{$k});
 132         }
 133
 134         pve_rados_connect($conn);
 135     };
 136     if (my $err = $@) {
 137         &$writedata($parent, 'E', $err);
 138         die $err;
 139     }
 140     &$writedata($parent, 'S');
 141
 142     $self->{conn} = $conn;
 143
 144     for (;;) {
 145         my ($cmd, $data) = &$readdata($parent, 1);
 146
 147         last if !$cmd || $cmd eq 'Q';
 148
 149         my $res;
 150         eval {
 151             if ($cmd eq 'M') { # rados monitor commands
 152                 $res = encode_json(pve_rados_mon_command($self->{conn}, [ $data ]));
 153             } elsif ($cmd eq 'C') { # class methods
 154                 my $aref = decode_json($data);
 155                 my $method = shift @$aref;
 156                 $res = encode_json($self->$method(@$aref));
 157             } else {
 158                 die "invalid command\n";
 159             }
 160         };
 161         if (my $err = $@) {
 162             &$writedata($parent, 'E', $err);
 163             die $err;
 164         }
 165         &$writedata($parent, '>', $res);
 166     }
 167 }
 168
 169 sub new {
 170     my ($class, %params) = @_;
 171
 172     my $rpcenv = PVE::RPCEnvironment::get();
 173
 174     socketpair(my $child, my $parent, AF_UNIX, SOCK_STREAM, PF_UNSPEC)
 175         ||  die "socketpair: $!\n";
 176
 177     my $cpid = fork();
 178
 179     die "unable to fork - $!\n" if !defined($cpid);
 180
 181     my $self = bless {}, $class;
 182
 183     my $timeout = delete $params{timeout} || $rados_default_timeout;
 184
 185     $self->{timeout} = $timeout;
 186     $self->{pid} = $$;
 187
 188     if ($cpid) { # parent
 189         close $parent;
 190
 191         $self->{cpid} = $cpid;
 192         $self->{child} = $child;
 193
 194         &$sendcmd($self, undef, undef, 'S'); # wait for sync
 195
 196     } else { # child
 197         $0 = 'pverados';
 198
 199         eval {
 200             PVE::INotify::inotify_close();
 201
 202             if (my $atfork = $rpcenv->{atfork}) {
 203                 &$atfork();
 204             }
 205
 206             # override signal handlers inherited from the parent
 207             local $SIG{HUP} = $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
 208                 pve_rados_shutdown($self->{conn}) if $self->{conn};
 209                 POSIX::_exit(1);
 210             };
 211
 212             # fixme: timeout?
 213
 214             close $child;
 215
 216             $self->pve_rados_work($parent, $timeout, %params);
 217         };
 218         my $err = $@;
 219         warn $err if $err;
 220
 221         pve_rados_shutdown($self->{conn}) if $self->{conn};
 222         POSIX::_exit($err ? 1 : 0);
 223     }
 224
 225     return $self;
 226 }
 227
 228 sub timeout {
 229     my ($self, $new_timeout) = @_;
 230
 231     $self->{timeout} = $new_timeout if $new_timeout;
 232
 233     return $self->{timeout};
 234 }
 235
 236 sub DESTROY {
 237     my ($self) = @_;
 238
 239     if ($self->{cpid}) {
 240         #print "$$: DESTROY WAIT0\n";
 241         &$kill_worker($self);
 242         #print "$$: DESTROY WAIT\n";
 243     }
 244 }
 245
 246 sub cluster_stat {
 247     my ($self, @args) = @_;
 248
 249     if ($self->{cpid}) {
 250         my $data = encode_json(['cluster_stat', @args]);
 251         my $raw = &$sendcmd($self, 'C', $data);
 252         return decode_json($raw);
 253     } else {
 254         return pve_rados_cluster_stat($self->{conn});
 255     }
 256 }
 257
 258 # example1: { prefix => 'get_command_descriptions'})
 259 # example2: { prefix => 'mon dump', format => 'json' }
 260 sub mon_cmd {
 261     my ($self, $cmd, $no_result_catch) = @_;
 262
 263     $cmd->{format} = 'json' if !$cmd->{format};
 264
 265     my $json = encode_json($cmd);
 266
 267     my $ret = $sendcmd->($self, 'M', $json);
 268     die "error with '$cmd->{prefix}': $@" if $@;
 269
 270     my $raw = decode_json($ret);
 271
 272     die "error with '$cmd->{prefix}': mon_cmd failed - $raw->{status_message}\n"
 273         if !$no_result_catch && $raw->{return_code} < 0;
 274
 275     my $data = '';
 276     if ($cmd->{format} && $cmd->{format} eq 'json') {
 277         $data = length($raw->{data}) ? decode_json($raw->{data}) : undef;
 278     } else {
 279         $data = $raw->{data};
 280     }
 281     return {
 282         return_code => $raw->{return_code},
 283         status_message => $raw->{status_message},
 284         data => $data,
 285     };
 286 }
 287
 288 # for backward compatibillity or if one just doesn't care about actual RADOS error details
 289 sub mon_command {
 290     my ($self, $cmd) = @_;
 291
 292     my $res = mon_cmd($self, $cmd);
 293
 294     return $res->{data};
 295 }
 296
 297 1;
 298 __END__
 299
 300 =head1 NAME
 301
 302 PVE::RADOS - Perl bindings for librados
 303
 304 =head1 SYNOPSIS
 305
 306   use PVE::RADOS;
 307
 308   my $rados = PVE::RADOS->new();
 309   my $stat = $rados->cluster_stat();
 310   my $res = $rados->mon_command({ prefix => 'mon dump', format => 'json' });
 311
 312 =head1 DESCRIPTION
 313
 314 Perl bindings for librados.
 315
 316 =head2 EXPORT
 317
 318 None by default.
 319
 320 =head1 AUTHOR
 321
 322 Dietmar Maurer, E<lt>dietmar@proxmox.com<gt>
 323 Thomas Lamprecht E<lt>t.lamprecht@proxmox.com<gt>
 324 Wolfgang Bumiller E<lt>w.bumiller@proxmox.com<gt>
 325
 326 =cut