always for worker process

librados does not work well when we fork after rados_connect().
So we always fork a separate worker and use IPC to send/receive data.

diff --git a/Makefile b/Makefile
index 550d0fce73dcd5a1509b63adf4c2965e915056b1..c67550a4e47800ed88620cae72dd7cf6c31d8dfc 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ RELEASE=3.1
 
 VERSION=0.72.1
 PACKAGE=librados2-perl
-PKGREL=1
+PKGREL=2
 
 DESTDIR=
 PREFIX=/usr

diff --git a/PVE/RADOS.pm b/PVE/RADOS.pm
index b736c046fdbfda250f38fc618bde053310de7ec8..a9d5dfd44ed4cad0642cb57a0c63f092a065aff5 100644 (file)
--- a/PVE/RADOS.pm
+++ b/PVE/RADOS.pm
@@ -5,7 +5,8 @@ use strict;
 use warnings;
 use Carp;
 use JSON;
-
+use Socket;
+ 
 require Exporter;
 
 our @ISA = qw(Exporter);
@@ -32,23 +33,126 @@ our $VERSION = '1.0';
 require XSLoader;
 XSLoader::load('PVE::RADOS', $VERSION);
 
-sub new {
-    my ($class, %params) = @_;
+# fixme: timeouts??
+
+my $writedata = sub {
+    my ($fh, $cmd, $data) = @_;
+
+    local $SIG{PIPE} = 'IGNORE';
+ 
+    my $bin = pack "a L/a*", $cmd, $data || '';
+    my $res = syswrite $fh, $bin;
+
+    die "write data failed - $!\n" if !defined($res);
+};
 
-    my $conn = pve_rados_create() ||
-       die "unable to create RADOS object\n";
+my $readdata = sub {
+    my ($fh, $expect_result) = @_;
 
-    my $timeout = delete $params{timeout} || 5;
+    my $head = '';
 
-    pve_rados_conf_set($conn, 'client_mount_timeout', $timeout);
+    local $SIG{PIPE} = 'IGNORE';
 
-    foreach my $k (keys %params) {
-       pve_rados_conf_set($conn, $k, $params{$k});
+    while (length($head) < 5) {
+       last if !sysread $fh, $head, 5 - length($head), length($head);
     }
+    die "partial read\n" if length($head) < 5;
+    
+    my ($cmd, $len) = unpack "a L", $head;
 
-    pve_rados_connect($conn);
+    my $data = '';
+    while (length($data) < $len) {
+       last if !sysread $fh, $data, $len - length($data), length($data);
+    }
+    die "partial data read\n" if length($data) < $len;
+
+    if ($expect_result) { 
+       die $data if $cmd eq 'E' && $data;
+       die "got unexpected result\n" if  $cmd ne '>';
+    }
 
-    my $self = bless { conn =>  $conn };
+    return wantarray ? ($cmd, $data) : $data;
+};
+
+sub new {
+    my ($class, %params) = @_;
+
+    socketpair(my $child, my $parent, AF_UNIX, SOCK_STREAM, PF_UNSPEC)
+       ||  die "socketpair: $!";
+
+    my $cpid = fork();
+
+    die "unable to fork - $!\n" if !defined($cpid);
+
+    my $self = bless {};
+
+    if ($cpid) { # parent
+       close $parent;
+ 
+       $self->{cpid} = $cpid;
+       $self->{child} = $child;
+
+       # wait for sync
+       my ($cmd, $msg) = &$readdata($child);
+       die $msg if $cmd eq 'E';
+       die "internal error- got unexpected result" if $cmd ne 'S';
+
+    } else { # child
+       $0 = 'pverados';
+ 
+       # fixme: timeout?
+
+       close $child;
+
+       my $timeout = delete $params{timeout} || 5;
+
+       my $conn;
+       eval {
+           $conn = pve_rados_create() ||
+               die "unable to create RADOS object\n";
+
+           pve_rados_conf_set($conn, 'client_mount_timeout', $timeout);
+
+           foreach my $k (keys %params) {
+               pve_rados_conf_set($conn, $k, $params{$k});
+           }
+
+           pve_rados_connect($conn);
+       };
+       if (my $err = $@) {
+           &$writedata($parent, 'E', $err);
+           die $err;
+       }
+       &$writedata($parent, 'S');
+
+       $self->{conn} = $conn;
+
+       for (;;) {
+           my ($cmd, $data) = &$readdata($parent);
+           
+           last if $cmd eq 'Q';
+
+           my $res;
+           eval {
+               if ($cmd eq 'M') { # rados monitor commands
+                   $res = pve_rados_mon_command($self->{conn}, [ $data ]);
+               } elsif ($cmd eq 'C') { # class methods
+                   my $aref = decode_json($data);
+                   my $method = shift @$aref;
+                   $res = encode_json($self->$method(@$aref));
+               } else {
+                   die "invalid command\n";
+               }
+           };
+           if (my $err = $@) {
+               &$writedata($parent, 'E', $err);
+               die $err;
+           }
+           &$writedata($parent, '>', $res);
+       }
+ 
+       exit(0);
+    }
 
     return $self;
 }
@@ -56,13 +160,27 @@ sub new {
 sub DESTROY {
     my ($self) = @_;
 
-    pve_rados_shutdown($self->{conn});
+    if ($self->{cpid}) {
+       #print "$$: DESTROY WAIT0\n";
+       eval { &$writedata($self->{child}, 'Q'); };
+       my $res = waitpid($self->{cpid}, 0);
+       #print "$$: DESTROY WAIT $res\n";
+    } else {
+       #print "$$: DESTROY SHUTDOWN\n";
+       pve_rados_shutdown($self->{conn}) if $self->{conn};
+    }
 }
 
 sub cluster_stat {
-    my ($self) = @_;
-
-    return  pve_rados_cluster_stat($self->{conn});
+    my ($self, @args) = @_;
+
+    if ($self->{cpid}) {
+       my $data = encode_json(['cluster_stat', @args]);
+       &$writedata($self->{child}, 'C', $data);
+       return decode_json(&$readdata($self->{child}, 1));
+    } else {
+       return pve_rados_cluster_stat($self->{conn});
+    }
 }
 
 # example1: { prefix => 'get_command_descriptions'})
@@ -73,7 +191,11 @@ sub mon_command {
     $cmd->{format} = 'json' if !$cmd->{format};
 
     my $json = encode_json($cmd);
-    my $raw = pve_rados_mon_command($self->{conn}, [ $json ]);
+
+    &$writedata($self->{child}, 'M', $json);
+
+    my $raw = &$readdata($self->{child}, 1);
+
     if ($cmd->{format} && $cmd->{format} eq 'json') {
        return length($raw) ? decode_json($raw) : undef;
     }

diff --git a/RADOS.xs b/RADOS.xs
index 438bcb716510687c9e7c375a80033e9ee161eb61..a9f6bc36c55337a9b0c565fc3819401a345c439c 100644 (file)
--- a/RADOS.xs
+++ b/RADOS.xs
@@ -54,9 +54,12 @@ CODE:
 {
     DPRINTF("pve_rados_connect\n");
 
-    rados_conf_read_file(cluster, NULL);
+    int res = rados_conf_read_file(cluster, NULL);
+    if (res < 0) {
+        die("rados_conf_read_file failed - %s\n", strerror(-res));
+    }
  
-    int res = rados_connect(cluster);
+    res = rados_connect(cluster);
     if (res < 0) {
         die("rados_connect failed - %s\n", strerror(-res));
     }

diff --git a/changelog.Debian b/changelog.Debian
index 6e557b53fce9df74e5f7969faa356e606a386681..bb23092463f56d14bef4ebf65068112b9d7de051 100644 (file)
--- a/changelog.Debian
+++ b/changelog.Debian
@@ -1,8 +1,8 @@
-libpve-storage-perl (3.0-19) unstable; urgency=low
+librados2-perl (0.72.1-2) unstable; urgency=low
 
-  * glusterfs : create volume through gluster block driver
+  * always fork worker process
 
- -- Proxmox Support Team <support@proxmox.com>  Thu, 16 Jan 2014 09:38:35 +0100
+ -- Proxmox Support Team <support@proxmox.com>  Mon, 20 Jan 2014 13:39:32 +0100
 
 librados2-perl (0.72.1-1) unstable; urgency=low