use strict;
use warnings;
-use POSIX qw(EINTR EEXIST EOPNOTSUPP);
-use IO::Socket::IP;
-use Socket qw(AF_INET AF_INET6 AI_ALL AI_V4MAPPED AI_CANONNAME SOCK_DGRAM IPPROTO_TCP);
-use IO::Select;
+
+use Date::Format qw(time2str);
+use Digest::MD5;
+use Digest::SHA;
+use Encode;
+use Fcntl qw(:DEFAULT :flock);
use File::Basename;
use File::Path qw(make_path);
use Filesys::Df (); # don't overwrite our df()
-use IO::Pipe;
-use IO::File;
use IO::Dir;
+use IO::File;
use IO::Handle;
+use IO::Pipe;
+use IO::Select;
+use IO::Socket::IP;
use IPC::Open3;
-use Fcntl qw(:DEFAULT :flock);
-use base 'Exporter';
-use URI::Escape;
-use Encode;
-use Digest::SHA;
use JSON;
-use Text::ParseWords;
+use POSIX qw(EINTR EEXIST EOPNOTSUPP);
+use Scalar::Util 'weaken';
+use Socket qw(AF_INET AF_INET6 AI_ALL AI_V4MAPPED AI_CANONNAME SOCK_DGRAM IPPROTO_TCP);
use String::ShellQuote;
+use Text::ParseWords;
use Time::HiRes qw(usleep gettimeofday tv_interval alarm);
-use Scalar::Util 'weaken';
-use Date::Format qw(time2str);
+use URI::Escape;
+use base 'Exporter';
use PVE::Syscall;
use constant {AT_EMPTY_PATH => 0x1000,
AT_FDCWD => -100};
+# from <linux/fs.h>
+use constant {RENAME_NOREPLACE => (1 << 0),
+ RENAME_EXCHANGE => (1 << 1),
+ RENAME_WHITEOUT => (1 << 2)};
+
sub run_with_timeout {
my ($timeout, $code, @param) = @_;
return "unable to read tail (got $br bytes)";
}
+# Check if the status returned by upid_read_status is an error status.
+# If the status could not be parsed it's also treated as an error.
+sub upid_status_is_error {
+ my ($status) = @_;
+
+ return !($status eq 'OK' || $status =~ m/^WARNINGS: \d+$/);
+}
+
# useful functions to store comments in config files
sub encode_text {
my ($text) = @_;
return 0 == syscall(PVE::Syscall::fsync, $fileno);
}
+sub renameat2($$$$$) {
+ my ($olddirfd, $oldpath, $newdirfd, $newpath, $flags) = @_;
+ return 0 == syscall(PVE::Syscall::renameat2, $olddirfd, $oldpath, $newdirfd, $newpath, $flags);
+}
+
sub sync_mountpoint {
my ($path) = @_;
sysopen my $fd, $path, O_RDONLY|O_CLOEXEC or die "failed to open $path: $!\n";
}
-# opts
-# -> hash_required
-# if 1, at least one checksum has to be specified otherwise an error will be thrown
-# -> http_proxy
-# -> https_proxy
-# -> verify_certificates
-# -> sha(1|224|256|384|512)sum
-# -> md5sum
+# opts is a hash ref with the following known properties
+# allow_overwrite - if 1, overwriting existing files is allowed, use with care. Default to false
+# hash_required - if 1, at least one checksum has to be specified otherwise an error will be thrown
+# http_proxy
+# https_proxy
+# verify_certificates - if 0 (false) we tell wget to ignore untrusted TLS certs. Default to true
+# md5sum|sha(1|224|256|384|512)sum - the respective expected checksum string
sub download_file_from_url {
my ($dest, $url, $opts) = @_;
- my $tmpdest = "$dest.tmp.$$";
-
- my $algorithm;
- my $expected;
-
+ my ($checksum_algorithm, $checksum_expected);
for ('sha512', 'sha384', 'sha256', 'sha224', 'sha1', 'md5') {
if (defined($opts->{"${_}sum"})) {
- $algorithm = $_;
- $expected = $opts->{"${_}sum"};
+ $checksum_algorithm = $_;
+ $checksum_expected = $opts->{"${_}sum"};
last;
}
}
+ die "checksum required but not specified\n" if ($opts->{hash_required} && !$checksum_algorithm);
- die "checksum required but not specified\n" if ($opts->{hash_required} && !$algorithm);
-
- my $worker = sub {
- my $upid = shift;
+ print "downloading $url to $dest\n";
- print "downloading $url to $dest\n";
+ if (-f $dest) {
+ if ($checksum_algorithm) {
+ print "calculating checksum of existing file...";
+ my $checksum_got = get_file_hash($checksum_algorithm, $dest);
- eval {
- if (-f $dest && $algorithm) {
- print "calculating checksum of existing file...\n";
- my $correct = check_file_hash($algorithm, $expected, $dest);
-
- if ($correct) {
- print "file already exists, no need to download\n";
- return;
- } else {
- print "mismatch, downloading\n";
- }
+ if (lc($checksum_got) eq lc($checksum_expected)) {
+ print "OK, got correct file already, no need to download\n";
+ return;
+ } elsif ($opts->{allow_overwrite}) {
+ print "checksum mismatch: got '$checksum_got' != expect '$checksum_expected', re-download\n";
+ } else {
+ print "\n"; # the front end expects the error to reside at the last line without any noise
+ die "checksum mismatch: got '$checksum_got' != expect '$checksum_expected', aborting\n";
}
+ } elsif (!$opts->{allow_overwrite}) {
+ die "refusing to override existing file '$dest'\n";
+ }
+ }
- my @cmd = ('/usr/bin/wget', '--progress=dot:mega', '-O', $tmpdest, $url);
+ my $tmpdest = "$dest.tmp.$$";
+ eval {
+ local $SIG{INT} = sub {
+ unlink $tmpdest or warn "could not cleanup temporary file: $!";
+ die "got interrupted by signal\n";
+ };
+ { # limit the scope of the ENV change
local %ENV;
if ($opts->{http_proxy}) {
$ENV{http_proxy} = $opts->{http_proxy};
$ENV{https_proxy} = $opts->{https_proxy};
}
- my $verify = $opts->{verify_certificates} // 1;
- if (!$verify) {
- push @cmd, '--no-check-certificate';
- }
+ my $cmd = ['wget', '--progress=dot:giga', '-O', $tmpdest, $url];
- if (run_command([[@cmd]]) != 0) {
- die "download failed: $!\n";
+ if (!($opts->{verify_certificates} // 1)) { # default to true
+ push @$cmd, '--no-check-certificate';
}
- if ($algorithm) {
- print "calculating checksum...\n";
+ run_command($cmd, errmsg => "download failed");
+ }
- my $correct = check_file_hash($algorithm, $expected, $tmpdest);
+ if ($checksum_algorithm) {
+ print "calculating checksum...";
- if ($correct) {
- print "checksum verified\n";
- } else {
- die "checksum mismatch\n";
- }
- } else {
- print "no checksum for verification specified\n";
- }
+ my $checksum_got = get_file_hash($checksum_algorithm, $tmpdest);
- if (!rename($tmpdest, $dest)) {
- die "unable to save file: $!\n";
+ if (lc($checksum_got) eq lc($checksum_expected)) {
+ print "OK, checksum verified\n";
+ } else {
+ print "\n"; # the front end expects the error to reside at the last line without any noise
+ die "checksum mismatch: got '$checksum_got' != expect '$checksum_expected'\n";
}
- };
- my $err = $@;
-
- unlink $tmpdest;
-
- if ($err) {
- print "\n";
- die $err;
}
- print "download finished\n";
+ rename($tmpdest, $dest) or die "unable to rename temporary file: $!\n";
};
+ if (my $err = $@) {
+ unlink $tmpdest or warn "could not cleanup temporary file: $!";
+ die $err;
+ }
- my $rpcenv = PVE::RPCEnvironment::get();
- my $user = $rpcenv->get_user();
-
- (my $filename = $dest) =~ s!.*/([^/]*)$!$1!;
-
- return $rpcenv->fork_worker('download', $filename, $user, $worker);
+ print "download of '$url' to '$dest' finished\n";
}
-sub check_file_hash {
- my ($algorithm, $expected, $filename) = @_;
+sub get_file_hash {
+ my ($algorithm, $filename) = @_;
my $algorithm_map = {
'md5' => sub { Digest::MD5->new },
my $digest = $digester->addfile($fh)->hexdigest;
- return lc($digest) eq lc($expected);
+ return lc($digest);
}
1;
projects / pve-common.git / blobdiff