-package Proxmox::Unpack;
+package PMG::Unpack;
use strict;
+use warnings;
use IO::File;
use IO::Select;
use Xdgmime;
-use Compress::Zlib qw (gzopen);
-use Compress::Bzip2 qw (bzopen);
+use Compress::Zlib qw(gzopen);
+use Compress::Bzip2 qw(bzopen);
use File::Path;
-use File::Temp qw (tempdir);
+use File::Temp qw(tempdir);
use File::Basename;
use File::stat;
use POSIX ":sys_wait_h";
-use Time::HiRes qw (usleep ualarm gettimeofday tv_interval);
+use Time::HiRes qw(usleep ualarm gettimeofday tv_interval);
use Archive::Zip qw(:CONSTANTS :ERROR_CODES);
use LibArchive;
-use Proxmox::Utils;
use MIME::Parser;
+use PMG::Utils;
+
my $unpackers = {
# TAR
#'application/x-cpio' => [ '7z', \&generic_unpack ],
# ZIP
- #'application/zip' => [ 'zip', \&unpack_tar, 1],
- #'application/zip' => [ 'zip', \&unpack_zip, 1],
- 'application/zip' => [ '7z', \&generic_unpack ],
+ #'application/zip' => [ 'zip', \&unpack_tar, 1],
+ 'application/zip' => [ '7z', \&generic_unpack ],
# 7z
'application/x-7z-compressed' => [ '7z', \&generic_unpack ],
# RAR
- 'application/x-rar' => [ '7z', \&generic_unpack ],
+ 'application/vnd.rar' => [ '7z', \&generic_unpack ],
# ARJ
'application/x-arj' => [ '7z', \&generic_unpack ],
'application/x-rpm' => [ '7z', \&generic_unpack ],
# DEB
- 'application/x-deb' => [ 'ar', \&unpack_tar, 1],
- #'application/x-deb' => [ '7z', \&generic_unpack ],
+ 'application/vnd.debian.binary-package' => [ 'ar', \&unpack_tar, 1],
# MS CAB
'application/vnd.ms-cab-compressed' => [ '7z', \&generic_unpack ],
};
my $decompressors = {
-
- 'application/x-gzip' => [ 'guzip', \&uncompress_file ],
+ 'application/gzip' => [ 'guzip', \&uncompress_file ],
'application/x-compress' => [ 'uncompress', \&uncompress_file ],
# 'application/x-compressed-tar' => [ 'guzip', \&uncompress_file ], # unpack_tar is faster
'application/x-tarz' => [ 'uncompress', \&uncompress_file ],
'application/x-bzip' => [ 'bunzip2', \&uncompress_file ],
'application/x-bzip-compressed-tar' => [ 'bunzip2', \&uncompress_file ],
-
};
my ($fh, $inputfilename, $errorfilename, @cmd) = @_;
my $pid = $fh->open ('-|');
-
+
die "unable to fork helper process: $!" if !defined $pid;
return $pid if ($pid != 0); # parent process simply returns
-
+
$inputfilename = '/dev/null' if !$inputfilename;
# same algorythm as used inside SA
POSIX::close(0) if $fd != 0;
if (!open (STDIN, "<$inputfilename")) {
- POSIX::_exit (1);
- kill ('KILL', $$);
+ POSIX::_exit (1);
+ kill ('KILL', $$);
}
$errorfilename = '&STDOUT' if !$errorfilename;
POSIX::close(2) if $fd != 2;
if (!open (STDERR, ">$errorfilename")) {
- POSIX::_exit (1);
- kill ('KILL', $$);
+ POSIX::_exit (1);
+ kill ('KILL', $$);
}
exec @cmd;
-
+
warn "exec failed";
- POSIX::_exit (1);
- kill('KILL', $$);
+ POSIX::_exit (1);
+ kill('KILL', $$);
die; # else -w complains
}
};
my $err = $@;
-
+
# send TERM first if process still exits
if ($err) {
kill (15, $pid) if kill (0, $pid);
# do nothing
}
}
-
+
# then close pipe
my $closeerr;
close ($cfh) || ($closeerr = $!);
};
my $err = $@;
-
+
alarm ($prev_alarm) if defined ($prev_alarm);
die "unknown error" if $sigcount && !$err; # seems to happen sometimes
$self->{maxrec} = $param{maxrec} || 8; # 0 = disabled
$self->{maxratio} = $param{maxratio} || 0; # 0 = disabled
- $self->{maxquota} = $param{quota} || 250*1024*1024; # 250 MB
+ $self->{maxquota} = $param{quota} || 250*1024*1024; # 250 MB
$self->{ctonly} = $param{ctonly}; # only detect contained content types
# internal
- $self->{quota} = 0;
+ $self->{quota} = 0;
$self->{ratioquota} = 0;
$self->{size} = 0;
$self->{files} = 0;
if ($app eq 'guzip' || $app eq 'bunzip2') {
my $cfh;
-
+
eval {
# bzip provides a gz compatible interface
- if ($app eq 'bunzip2') {
+ if ($app eq 'bunzip2') {
$self->{mime}->{'application/x-bzip'} = 1;
$cfh = bzopen ("$filename", 'r');
die "bzopen '$filename' failed" if !$cfh;
} else {
- $self->{mime}->{'application/x-gzip'} = 1;
+ $self->{mime}->{'application/gzip'} = 1;
$cfh = gzopen ("$filename", 'rb');
die "gzopen '$filename' failed" if !$cfh;
}
-
+
run_with_timeout ($timeout, sub {
my $count;
my $buf;
});
};
-
+
$err = $@;
}
sub check_timeout {
my ($self) = @_;
-
+
my $elapsed = int (tv_interval ($self->{starttime}));
my $timeout = $self->{timeout} - $elapsed;
die "compresion ratio too large (> $self->{maxratio})"
if $self->{maxratio} && (($self->{size} + $sizediff) > $self->{ratioquota});
- die "archive too large (> $self->{quota})"
+ die "archive too large (> $self->{quota})"
if ($self->{size} + $sizediff) > $self->{quota};
die "unexpected number of files '$files'" if $files <= 0;
$self->{files} += $files;
$self->{size} += $sizediff;
}
-
+
}
sub add_glob_mime_type {
eval {
run_with_timeout ($timeout, sub {
-
+
# Create a new MIME parser:
my $parser = new MIME::Parser;
$parser->output_under ($tmpdir);
- $parser->extract_nested_messages (1);
+ $parser->extract_nested_messages (1);
$parser->ignore_errors (1);
$parser->extract_uuencode (1);
$parser->filer->ignore_filename(1);
my $entity = $parser->parse_open ($filename);
($files, $size) = $self->walk_mime_entity ($entity);
-
+
});
};
my $status = $zip->read ($filename);
die "unable to open zip file '$filename'" if $status != AZ_OK;
-
+
my $tid = 1;
foreach my $mem ($zip->members) {
my $cm = $mem->compressionMethod();
die "unsupported zip compression method '$cm'\n"
- if !(($cm == COMPRESSION_DEFLATED ||
+ if !(($cm == COMPRESSION_DEFLATED ||
$cm == COMPRESSION_STORED));
die "encrypted archive detected\n"
my $outfd = IO::File->new;
if (!$outfd->open ($newfn, O_CREAT|O_EXCL|O_WRONLY, 0640)) {
- die "unable to create file $newfn: $!";
+ die "unable to create file $newfn: $!";
}
my $ct;
my $bytes = 0;
while ($status == AZ_OK) {
($outRef, $status) = $mem->readChunk();
- die "unable to read zip member"
+ die "unable to read zip member"
if ($status != AZ_OK && $status != AZ_STREAM_END);
my $len = length ($$outRef);
last if $status == AZ_STREAM_END;
}
- $mem->endRead();
+ $mem->endRead();
$self->todo_list_add ($newfn, $ct, $bytes);
run_with_timeout ($timeout, sub {
if ((my $r = LibArchive::archive_read_open_filename ($a, $filename, 10240))) {
- die "LibArchive error: %s", LibArchive::archive_error_string ($a);
+ die "LibArchive error: %s", LibArchive::archive_error_string ($a);
}
my $tid = 1;
for (;;) {
my $len;
my $buf;
while (($len = LibArchive::archive_read_data($a, $buf, 128*1024)) > 0) {
-
+
if (!$bytes) {
if ($ct = xdg_mime_get_mime_type_for_data ($buf, $len)) {
$self->{mime}->{$ct} = 1;
}
}
}
-
+
$bytes += $len;
if (!$outfd) { # create only when needed
$outfd = IO::File->new;
-
+
if (!$outfd->open ($newfn, O_CREAT|O_EXCL|O_WRONLY, 0640)) {
die "unable to create file $newfn: $!";
}
my $err = $@;
$outfd->close () if $outfd;
-
+
if ($err) {
unlink $newfn;
die $err;
}
- }
+ }
});
};
LibArchive::archive_read_close($a);
LibArchive::archive_read_finish($a);
-
+
die $err if $err;
$self->check_quota ($files, $size, $csize, 1); # commit sizes
my @listcmd;
my @restorecmd = ('/bin/false');
- my $filter;
+ my $filter;
if ($app eq 'tar') {
@listcmd = ('/bin/tar', '-tvf', $filename);
undef $path;
undef $folder;
undef $bytes;
-
+
} elsif ($line =~ m/^Path = (.*)\z/s) {
$path = $1;
} elsif ($line =~ m/^Size = (\d+)\z/s) {
my $cfh = IO::File->new();
my $pid = helper_pipe_open ($cfh, '/dev/null', '/dev/null', @listcmd);
-
+
helper_pipe_consume ($cfh, $pid, $timeout, 0, $filter);
};
my $name;
- while (defined ($name = readdir (DIR))) {
+ while (defined ($name = readdir (DIR))) {
my $path = "$dirname/$name";
my $st = lstat ($path);
$level = 0 if !$level;
- $self->{levels} = max2 ($self->{levels}, $level);
+ $self->{levels} = max2($self->{levels}, $level);
if ($self->{maxrec} && ($level >= $self->{maxrec})) {
- return if $self->{maxrec_soft};
+ return if $self->{maxrec_soft};
die "max recursion limit reached\n";
}
return if !$size; # nothing to do
if (!$ct) {
- $ct = Proxmox::Utils::magic_mime_type_for_file ($filename);
- $self->add_glob_mime_type ($filename);
+ $ct = PMG::Utils::magic_mime_type_for_file($filename);
+ $self->add_glob_mime_type($filename);
}
if ($ct) {
$self->{mime}->{$ct} = 1;
- if (defined ($decompressors->{$ct})) {
+ if (defined($decompressors->{$ct})) {
my ($app, $code) = @{$decompressors->{$ct}};
if ($app) {
# we try to keep extension correctly
- my $tmp = basename ($filename);
- ($ct eq 'application/x-gzip') &&
+ my $tmp = basename($filename);
+ ($ct eq 'application/gzip') &&
$tmp =~ s/\.gz\z//;
- ($ct eq 'application/x-bzip') &&
+ ($ct eq 'application/x-bzip') &&
$tmp =~ s/\.bz2?\z//;
- ($ct eq 'application/x-compress') &&
+ ($ct eq 'application/x-compress') &&
$tmp =~ s/\.Z\z//;
- ($ct eq 'application/x-compressed-tar') &&
+ ($ct eq 'application/x-compressed-tar') &&
$tmp =~ s/\.gz\z// || $tmp =~ s/\.tgz\z/.tar/;
- ($ct eq 'application/x-bzip-compressed-tar') &&
+ ($ct eq 'application/x-bzip-compressed-tar') &&
$tmp =~ s/\.bz2?\z// || $tmp =~ s/\.tbz\z/.tar/;
($ct eq 'application/x-tarz') &&
$tmp =~ s/\.Z\z//;
- my $newname = sprintf "%s/DC_%08d_%s", $self->{tmpdir}, ++$self->{ufid}, $tmp;
+ my $newname = sprintf "%s/DC_%08d_%s", $self->{tmpdir}, ++$self->{ufid}, $tmp;
print "Decomp: $filename\n\t($ct) with $app to $newname\n"
- if $self->{debug};
+ if $self->{debug};
- if (my $res = &$code ($self, $app, $filename, $newname, $level ? $size : 0, $size)) {
+ if (my $res = &$code($self, $app, $filename, $newname, $level ? $size : 0, $size)) {
unlink $filename if $level;
$self->unpack_todo ($level + 1);
}
if ($app) {
- my $tmpdir = sprintf "%s/DIR_%08d", $self->{tmpdir}, ++$self->{udid};
+ my $tmpdir = sprintf "%s/DIR_%08d", $self->{tmpdir}, ++$self->{udid};
mkdir $tmpdir;
-
+
print "Unpack: $filename\n\t($ct) with $app to $tmpdir\n"
if $self->{debug};
}
sub is_archive {
- my $ct = shift;
+ my ($ct) = @_;
- return defined ($decompressors->{$ct}) || defined ($unpackers->{$ct});
+ return defined($decompressors->{$ct}) || defined($unpackers->{$ct});
}
# unpack_archive
#
# Description: unpacks an archive and records containing
# content types (detected by magic numbers and file extension)
-# Extracted files are stored inside 'tempdir'.
-#
+# Extracted files are stored inside 'tempdir'.
+#
# returns: true if file is archive, undef otherwhise
sub unpack_archive {
my ($self, $filename, $ct) = @_;
- my $st = lstat ($filename);
+ my $st = lstat($filename);
my $size = 0;
if (!$st) {
die "no such file '$filename' - $!";
- } elsif (POSIX::S_ISREG ($st->mode)) {
+ } elsif (POSIX::S_ISREG($st->mode)) {
$size = $st->size;
return if !$size; # do nothing
$self->{quota} = $self->{maxquota} - $self->{size};
$self->{ratioquota} = $size * $self->{maxratio} if $self->{maxratio};
-
+
} else {
return; # do nothing
}
-
- $ct = Proxmox::Utils::magic_mime_type_for_file ($filename) if !$ct;
- return if (!$ct || !is_archive ($ct)); # not an archive
+ $ct = PMG::Utils::magic_mime_type_for_file($filename) if !$ct;
+
+ return if (!$ct || !is_archive($ct)); # not an archive
eval {
- $self->__unpack_archive ($filename, 0, $st->size, $ct);
+ $self->__unpack_archive($filename, 0, $st->size, $ct);
};
my $err = $@;
- printf "ELAPSED: %.2f ms $filename\n",
+ printf "ELAPSED: %.2f ms $filename\n",
int(tv_interval ($self->{starttime}) * 1000)
if $self->{debug};
if ($err) {
- $self->{mime}->{'proxmox/unreadable-archive'} = 1;
+ $self->{mime}->{'proxmox/unreadable-archive'} = 1;
die $err;
}
return 1;
projects / pmg-api.git / blobdiff