From d93f8aea9d49763f55f788385d0d4375ef866699 Mon Sep 17 00:00:00 2001 From: Max Carrara Date: Tue, 2 Apr 2024 16:55:18 +0200 Subject: [PATCH 01/16] cephconfig: align written key-value pairs by tab instead of tab + space. Signed-off-by: Max Carrara Tested-by: Friedrich Weber --- src/PVE/CephConfig.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PVE/CephConfig.pm b/src/PVE/CephConfig.pm index 1b6e86c..845b7d2 100644 --- a/src/PVE/CephConfig.pm +++ b/src/PVE/CephConfig.pm @@ -72,7 +72,7 @@ sub write_ceph_config { $out .= "[$section]\n"; for my $key (sort keys $cfg->{$section}->%*) { - $out .= "\t $key = $cfg->{$section}->{$key}\n"; + $out .= "\t$key = $cfg->{$section}->{$key}\n"; } $out .= "\n"; -- 2.39.2 From ad6bcc9e00808b7848fd2caca47525a191b24ff1 Mon Sep 17 00:00:00 2001 From: Max Carrara Date: Tue, 2 Apr 2024 16:55:19 +0200 Subject: [PATCH 02/16] cephconfig: escape un-escaped comment literals on write in order to prevent configuration errors or the configuration being misinterpreted. Signed-off-by: Max Carrara Tested-by: Friedrich Weber --- src/PVE/CephConfig.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/PVE/CephConfig.pm b/src/PVE/CephConfig.pm index 845b7d2..0def1f2 100644 --- a/src/PVE/CephConfig.pm +++ b/src/PVE/CephConfig.pm @@ -105,6 +105,9 @@ sub write_ceph_config { $cond_write_sec->($re); } + # Escape comment literals that aren't escaped already + $out =~ s/(? Date: Tue, 2 Apr 2024 16:55:20 +0200 Subject: [PATCH 03/16] cephconfig: align our parser with Ceph's parser This commit rewrites the entire parser for ceph.conf, aligning its behaviour as closely as possible with Ceph's parser grammar [0]. The most notable improvements are as follows: 1. The characters '#' and ';' now both mark comments, instead of just the '#' character. 2. Any character, including comment literals ('#' and ';'), may now be escaped. 3. Quoted values (single and double) are now supported. 4. Line continuations are now supported (lines ending with '\'). 5. Repeated whitespace characters in keys are now treated as a single space character. 6. Dashes '-' are not treated the same as spaces and underscores anymore, as Ceph's grammar doesn't treat them that way. * Paired with 5., this means that repeated whitespace is now equivalent to a single underscore. 7. Escaped comment literals are now un-escaped. 8. Although not too crucial, the parser now also supports empty sections and will just initialize them with an empty hash. Furthermore, the original grammar's more quirky behaviours are also respected where sanely possible. [0]: https://git.proxmox.com/?p=ceph.git;a=blob;f=ceph/src/common/ConfUtils.cc;h=2f78fd02bf9e27467275752e6f3bca0c5e3946ce;hb=e9fe820e7fffd1b7cde143a9f77653b73fcec748#l144 Signed-off-by: Max Carrara Tested-by: Friedrich Weber --- src/PVE/CephConfig.pm | 236 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 222 insertions(+), 14 deletions(-) diff --git a/src/PVE/CephConfig.pm b/src/PVE/CephConfig.pm index 0def1f2..56e7989 100644 --- a/src/PVE/CephConfig.pm +++ b/src/PVE/CephConfig.pm @@ -10,36 +10,244 @@ cfs_register_file('ceph.conf', \&parse_ceph_config, \&write_ceph_config); +# For more information on how the Ceph parser works and how its grammar is +# defined, see: +# https://git.proxmox.com/?p=ceph.git;a=blob;f=ceph/src/common/ConfUtils.cc;h=2f78fd02bf9e27467275752e6f3bca0c5e3946ce;hb=e9fe820e7fffd1b7cde143a9f77653b73fcec748#l144 sub parse_ceph_config { my ($filename, $raw) = @_; my $cfg = {}; return $cfg if !defined($raw); - my @lines = split /\n/, $raw; + # Note: According to Ceph's config grammar, a single key-value pair in a file + # (and nothing else!) is a valid config file and will be parsed by `ceph-conf`. + # We choose not to handle this case here because it doesn't seem to be used + # by Ceph at all (and otherwise doesn't really make sense anyway). + + # Regexes ending with '_class' consist of only an extended character class + # each, which allows them to be interpolated into other ext. char classes. + + my $re_leading_ws = qr/^\s+/; + my $re_trailing_ws = qr/\s+$/; + + my $re_continue_marker = qr/\\/; + my $re_comment_class = qr/(?[ [ ; # ] ])/; + my $re_not_allowed_in_section_header_class = qr/(?[ [ \] ] + $re_comment_class ])/; + + # Note: The Ceph config grammar defines keys the following way: + # + # key %= raw[+(text_char - char_("=[ ")) % +blank]; + # + # The ' - char_("=[ ")' expression might lure you into thinking that keys + # may *not* contain spaces, but they can, due to the "% +blank" at the end! + # + # See: https://www.boost.org/doc/libs/1_42_0/libs/spirit/doc/html/spirit/qi/reference/operator/list.html + # + # Allowing spaces in this class and later squeezing whitespace as well as + # removing any leading and trailing whitespace from keys is just so much + # easier in our case. + my $re_not_allowed_in_keys_class = qr/(?[ [ = \[ ] + $re_comment_class ])/; + + my $re_not_allowed_in_single_quoted_text_class = qr/(?[ [ ' ] + $re_comment_class ])/; + my $re_not_allowed_in_double_quoted_text_class = qr/(?[ [ " ] + $re_comment_class ])/; + + my $re_text_char = qr/\\.|(?[ ! $re_comment_class ])/; + my $re_section_header_char = qr/\\.|(?[ ! $re_not_allowed_in_section_header_class ])/; + + my $re_key_char = qr/\\.|(?[ ! $re_not_allowed_in_keys_class ])/; + + my $re_single_quoted_text_char = qr/\\.|(?[ ! $re_not_allowed_in_single_quoted_text_class ])/; + my $re_double_quoted_text_char = qr/\\.|(?[ ! $re_not_allowed_in_double_quoted_text_class ])/; + + my $re_single_quoted_value = qr/'(($re_single_quoted_text_char)*)'/; + my $re_double_quoted_value = qr/"(($re_double_quoted_text_char)*)"/; + + my $re_key = qr/^(($re_key_char)+)/; + my $re_quoted_value = qr/$re_single_quoted_value|$re_double_quoted_value/; + my $re_unquoted_value = qr/(($re_text_char)*)/; + my $re_value = qr/($re_quoted_value|$re_unquoted_value)/; + + my $re_kv_separator = qr/\s*(=)\s*/; + + my $re_section_start = qr/\[/; + my $re_section_end = qr/\]/; + my $re_section_header = qr/$re_section_start(($re_section_header_char)+)$re_section_end/; my $section; + my @lines = split(/\n/, $raw); + + my $parse_section_header = sub { + my ($section_line) = @_; + + # continued lines in section headers are allowed + while ($section_line =~ s/$re_continue_marker$//) { + $section_line .= shift(@lines); + } + + my $remainder = $section_line; + + $remainder =~ s/$re_section_header//; + my $parsed_header = $1; + + # Un-escape comment literals + $parsed_header =~ s/\\($re_comment_class)/$1/g; + + if (!$parsed_header) { + die "failed to parse section - skip: $section_line\n"; + } + + # preserve Ceph's behaviour and disallow anything after the section header + # that's not whitespace or a comment + $remainder =~ s/$re_leading_ws//; + $remainder =~ s/^$re_comment_class.*$//; + + if ($remainder) { + die "unexpected remainder after section - skip: $section_line\n"; + } + + return $parsed_header; + }; + + my $parse_key = sub { + my ($line) = @_; + + my $remainder = $line; + + my $key = ''; + while ($remainder =~ s/$re_key//) { + $key .= $1; + + while ($key =~ s/$re_continue_marker$//) { + $remainder = shift(@lines); + } + } + + $key =~ s/$re_trailing_ws//; + $key =~ s/$re_leading_ws//; + + $key =~ s/\s/ /; + while ($key =~ s/\s\s/ /) {} # squeeze repeated whitespace + + # Ceph treats *single* spaces in keys the same as underscores, + # but we'll just use underscores for readability + $key =~ s/ /_/g; + + # Un-escape comment literals + $key =~ s/\\($re_comment_class)/$1/g; + + if ($key eq '') { + die "failed to parse key from line - skip: $line\n"; + } + + my $had_equals = $remainder =~ s/^$re_kv_separator//; + + if (!$had_equals) { + die "expected '=' after key - skip: $line\n"; + } + + while ($remainder =~ s/^$re_continue_marker$//) { + # Whitespace and continuations after equals sign can be arbitrary + $remainder = shift(@lines); + $remainder =~ s/$re_leading_ws//; + } + + return ($key, $remainder); + }; + + my $parse_value = sub { + my ($line, $remainder) = @_; + + my $starts_with_quote = $remainder =~ m/^['"]/; + $remainder =~ s/$re_value//; + my $value = $1 // ''; + + if ($value eq '') { + die "failed to parse value - skip: $line\n"; + } + + if ($starts_with_quote) { + # If it started with a quote, the parsed value MUST end with a quote + my $is_single_quoted = $value =~ m/$re_single_quoted_value/; + $value = $1 if $is_single_quoted; + my $is_double_quoted = !$is_single_quoted && $value =~ m/$re_double_quoted_value/; + $value = $1 if $is_double_quoted; - foreach my $line (@lines) { - $line =~ s/#.*$//; - $line =~ s/^\s+//; - $line =~ s/^;.*$//; - $line =~ s/\s+$//; + if (!($is_single_quoted || $is_double_quoted)) { + die "failed to parse quoted value - skip: $line\n"; + } + + # Optionally, *only* line continuations may *only* follow right after + while ($remainder =~ s/^$re_continue_marker$//) { + $remainder .= shift(@lines); + } + + # Nothing but whitespace or a comment may follow + $remainder =~ s/$re_leading_ws//; + $remainder =~ s/^$re_comment_class.*$//; + + if ($remainder) { + die "unexpected remainder after value - skip: $line\n"; + } + + } else { + while ($value =~ s/$re_continue_marker$//) { + my $next_line = shift(@lines); + + $next_line =~ s/$re_unquoted_value//; + my $value_part = $1 // ''; + $value .= $value_part; + } + + $value =~ s/$re_trailing_ws//; + } + + # Un-escape comment literals + $value =~ s/\\($re_comment_class)/$1/g; + + return $value; + }; + + while (scalar(@lines)) { + my $line = shift(@lines); + + $line =~ s/^\s*(?($line) }; + if ($@) { + warn "$@\n"; + } - $section = $1 if $line =~ m/^\[(\S+)\]$/; - if (!$section) { - warn "no section - skip: $line\n"; + if (defined($section)) { + $cfg->{$section} = {} if !exists($cfg->{$section}); + } + + next; + } + + if (!defined($section)) { + warn "no section header - skip: $line\n"; next; } - if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) { - my ($key, $val) = ($1, $2); - # ceph treats ' ', '_' and '-' in keys the same, so lets do too - $key =~ s/[-\ ]/_/g; - $cfg->{$section}->{$key} = $val; + my ($key, $remainder) = eval { $parse_key->($line) }; + if ($@) { + warn "$@\n"; + next; + } + + my $value = eval { $parse_value->($line, $remainder) }; + if ($@) { + warn "$@\n"; + next; } + $cfg->{$section}->{$key} = $value; } return $cfg; -- 2.39.2 From d47460eec7d2c5eaebf94be7fbe072be37780520 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fabian=20Gr=C3=BCnbichler?= Date: Thu, 11 Apr 2024 14:33:56 +0200 Subject: [PATCH 04/16] bump version to 8.1.5 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Fabian Grünbichler --- debian/changelog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/debian/changelog b/debian/changelog index a9ee916..aef5f84 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +libpve-storage-perl (8.1.5) bookworm; urgency=medium + + * esxi: status: mark as active if its mounted + + * esxi: add mapping for windows server 2016/2019 + + * ceph config: rewrite parser/writer, allow arbitrary sections + + -- Proxmox Support Team Thu, 11 Apr 2024 11:56:49 +0200 + libpve-storage-perl (8.1.4) bookworm; urgency=medium * esxi: add optional 'port' config parameter -- 2.39.2 From b6fc9de14ad8da71037f9bd3d63055cc1f517416 Mon Sep 17 00:00:00 2001 From: Mira Limbeck Date: Wed, 17 Apr 2024 11:48:57 +0200 Subject: [PATCH 05/16] fix insecure migration failing if waiting on lock both STDOUT and STDERR are written into `$info` which is then parsed for IP and port of the target socket listening. when the ports file can't be locked immediately `trying to acquire lock...` is printed on STDERR and in turn written into `$info`. trying to parse the IP then fails, resulting in a migration or replication failing. the bare open3 call is replaced by the run_command wrapper from pve-common to use a safe wrapper around open3 with the same functionality. STDERR is read separatey from STDOUT and the last line of STDERR is kept in case of errors. Fixes: 57acd6a ("fix #1452: also log stderr of remote command with insecure storage migration") Signed-off-by: Mira Limbeck --- src/PVE/Storage.pm | 91 ++++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm index 40314a8..ade4523 100755 --- a/src/PVE/Storage.pm +++ b/src/PVE/Storage.pm @@ -851,43 +851,62 @@ sub storage_migrate { eval { if ($insecure) { - my $input = IO::File->new(); - my $info = IO::File->new(); - open3($input, $info, $info, @$recv) - or die "receive command failed: $!\n"; - close($input); - - my $try_ip = <$info> // ''; - my ($ip) = $try_ip =~ /^($PVE::Tools::IPRE)$/ # untaint - or die "no tunnel IP received, got '$try_ip'\n"; - - my $try_port = <$info> // ''; - my ($port) = $try_port =~ /^(\d+)$/ # untaint - or die "no tunnel port received, got '$try_port'\n"; - - my $socket = IO::Socket::IP->new(PeerHost => $ip, PeerPort => $port, Type => SOCK_STREAM) - or die "failed to connect to tunnel at $ip:$port\n"; - # we won't be reading from the socket - shutdown($socket, 0); - - eval { run_command($cmds, output => '>&'.fileno($socket), errfunc => $match_volid_and_log); }; - my $send_error = $@; - - # don't close the connection entirely otherwise the receiving end - # might not get all buffered data (and fails with 'connection reset by peer') - shutdown($socket, 1); - - # wait for the remote process to finish - while (my $line = <$info>) { - $match_volid_and_log->("[$target_sshinfo->{name}] $line"); - } + my $ip; + my $port; + my $socket; + my $send_error; + + my $handle_insecure_migration = sub { + my $line = shift; + + if (!$ip) { + ($ip) = $line =~ /^($PVE::Tools::IPRE)$/ # untaint + or die "no tunnel IP received, got '$line'\n"; + } elsif (!$port) { + ($port) = $line =~ /^(\d+)$/ # untaint + or die "no tunnel port received, got '$line'\n"; + + # create socket, run command + $socket = IO::Socket::IP->new( + PeerHost => $ip, + PeerPort => $port, + Type => SOCK_STREAM, + ); + die "failed to connect to tunnel at $ip:$port\n" if !$socket; + # we won't be reading from the socket + shutdown($socket, 0); + + eval { + run_command( + $cmds, + output => '>&'.fileno($socket), + errfunc => $match_volid_and_log, + ); + }; + $send_error = $@; + + # don't close the connection entirely otherwise the receiving end + # might not get all buffered data (and fails with 'connection reset by peer') + shutdown($socket, 1); + } else { + $match_volid_and_log->("[$target_sshinfo->{name}] $line"); + } + }; - # now close the socket - close($socket); - if (!close($info)) { # does waitpid() - die "import failed: $!\n" if $!; - die "import failed: exit code ".($?>>8)."\n"; - } + eval { + run_command( + $recv, + outfunc => $handle_insecure_migration, + errfunc => sub { + my $line = shift; + + $match_volid_and_log->("[$target_sshinfo->{name}] $line"); + }, + ); + }; + my $recv_err = $@; + close($socket) if $socket; + die "failed to run insecure migration: $recv_err\n" if $recv_err; die $send_error if $send_error; } else { -- 2.39.2 From 9e222078a9ccefc67d79e1e55fe62deec3432e5f Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Wed, 17 Apr 2024 12:20:36 +0200 Subject: [PATCH 06/16] storage migrate: tiny line reduction Signed-off-by: Thomas Lamprecht --- src/PVE/Storage.pm | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm index ade4523..1dc79ae 100755 --- a/src/PVE/Storage.pm +++ b/src/PVE/Storage.pm @@ -851,9 +851,7 @@ sub storage_migrate { eval { if ($insecure) { - my $ip; - my $port; - my $socket; + my ($ip, $port, $socket); my $send_error; my $handle_insecure_migration = sub { @@ -899,7 +897,6 @@ sub storage_migrate { outfunc => $handle_insecure_migration, errfunc => sub { my $line = shift; - $match_volid_and_log->("[$target_sshinfo->{name}] $line"); }, ); -- 2.39.2 From 3e5f91823a56e74ac629f85d7b6f1b6077a40a6b Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Wed, 17 Apr 2024 12:21:08 +0200 Subject: [PATCH 07/16] storage migrate: only output about cleaning-up snapshots if there are any Signed-off-by: Thomas Lamprecht --- src/PVE/Storage.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm index 1dc79ae..f19a115 100755 --- a/src/PVE/Storage.pm +++ b/src/PVE/Storage.pm @@ -915,8 +915,8 @@ sub storage_migrate { if !defined($new_volid) && $target_apiver >= 5; }; my $err = $@; - warn "send/receive failed, cleaning up snapshot(s)..\n" if $err; if ($opts->{migration_snapshot}) { + warn "send/receive failed, cleaning up snapshot(s)..\n" if $err; eval { volume_snapshot_delete($cfg, $volid, $opts->{snapshot}, 0) }; warn "could not remove source snapshot: $@\n" if $@; } -- 2.39.2 From f8a368c63c79626544cd01fbcd1028eae79b6128 Mon Sep 17 00:00:00 2001 From: Hannes Duerr Date: Tue, 19 Dec 2023 15:03:06 +0100 Subject: [PATCH 08/16] fix #1611: implement import of base-images for LVM-thin Storage for base images we call the volume_import of the parent plugin and pass it as vm-image instead of base-image, then convert it back as base-image Signed-off-by: Hannes Duerr --- src/PVE/Storage/LvmThinPlugin.pm | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/PVE/Storage/LvmThinPlugin.pm b/src/PVE/Storage/LvmThinPlugin.pm index 1d2e37c..96f619b 100644 --- a/src/PVE/Storage/LvmThinPlugin.pm +++ b/src/PVE/Storage/LvmThinPlugin.pm @@ -383,6 +383,56 @@ sub volume_has_feature { return undef; } +sub volume_import { + my ($class, $scfg, $storeid, $fh, $volname, $format, $snapshot, $base_snapshot, $with_snapshots, $allow_rename) = @_; + + my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $file_format) = + $class->parse_volname($volname); + + if (!$isBase) { + return $class->SUPER::volume_import( + $scfg, + $storeid, + $fh, + $volname, + $format, + $snapshot, + $base_snapshot, + $with_snapshots, + $allow_rename + ); + } else { + my $tempname; + my $vg = $scfg->{vgname}; + my $lvs = PVE::Storage::LVMPlugin::lvm_list_volumes($vg); + if ($lvs->{$vg}->{$volname}) { + die "volume $vg/$volname already exists\n" if !$allow_rename; + warn "volume $vg/$volname already exists - importing with a different name\n"; + + $tempname = $class->find_free_diskname($storeid, $scfg, $vmid); + } else { + $tempname = $volname; + $tempname =~ s/base/vm/; + } + + ($storeid,my $newname) = PVE::Storage::parse_volume_id($class->SUPER::volume_import( + $scfg, + $storeid, + $fh, + $tempname, + $format, + $snapshot, + $base_snapshot, + $with_snapshots, + $allow_rename + )); + + $volname = $class->create_base($storeid, $scfg, $newname); + } + + return "$storeid:$volname"; +} + # used in LVMPlugin->volume_import sub volume_import_write { my ($class, $input_fh, $output_file) = @_; -- 2.39.2 From a12671ae979148259090005591ea0f945bbe8d5a Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fabian=20Gr=C3=BCnbichler?= Date: Thu, 18 Apr 2024 15:10:46 +0200 Subject: [PATCH 09/16] lvmthin: import: improve readability MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit the nested call combined with the long parameter list is not very nice to format/read, let's split it.. Signed-off-by: Fabian Grünbichler --- src/PVE/Storage/LvmThinPlugin.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/PVE/Storage/LvmThinPlugin.pm b/src/PVE/Storage/LvmThinPlugin.pm index 96f619b..4b23623 100644 --- a/src/PVE/Storage/LvmThinPlugin.pm +++ b/src/PVE/Storage/LvmThinPlugin.pm @@ -415,7 +415,7 @@ sub volume_import { $tempname =~ s/base/vm/; } - ($storeid,my $newname) = PVE::Storage::parse_volume_id($class->SUPER::volume_import( + my $newvolid = $class->SUPER::volume_import( $scfg, $storeid, $fh, @@ -425,7 +425,8 @@ sub volume_import { $base_snapshot, $with_snapshots, $allow_rename - )); + ); + ($storeid,my $newname) = PVE::Storage::parse_volume_id($newvolid); $volname = $class->create_base($storeid, $scfg, $newname); } -- 2.39.2 From 0cb758c919111c6f51d7a8592a06ecffb350434c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fabian=20Gr=C3=BCnbichler?= Date: Thu, 11 Jan 2024 11:51:23 +0100 Subject: [PATCH 10/16] upload: use SSH helper to get ssh/scp options MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Fabian Grünbichler --- src/PVE/API2/Storage/Status.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm index f7e324f..dc6cc69 100644 --- a/src/PVE/API2/Storage/Status.pm +++ b/src/PVE/API2/Storage/Status.pm @@ -467,9 +467,9 @@ __PACKAGE__->register_method ({ if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) { my $remip = PVE::Cluster::remote_node_ip($node); - my @ssh_options = ('-o', 'BatchMode=yes'); + my $ssh_options = PVE::SSHInfo::ssh_info_to_ssh_opts({ ip => $remip, name => $node }); - my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--'); + my @remcmd = ('/usr/bin/ssh', $ssh_options->@*, $remip, '--'); eval { # activate remote storage run_command([@remcmd, '/usr/sbin/pvesm', 'status', '--storage', $param->{storage}]); @@ -481,7 +481,7 @@ __PACKAGE__->register_method ({ errmsg => "mkdir failed", ); - $cmd = ['/usr/bin/scp', @ssh_options, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)]; + $cmd = ['/usr/bin/scp', $ssh_options->@*, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)]; $err_cleanup = sub { run_command([@remcmd, 'rm', '-f', '--', $dest]) }; } else { -- 2.39.2 From 0947165111d730338ccf70cf20ce12d4415d7716 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Fri, 19 Apr 2024 08:41:21 +0200 Subject: [PATCH 11/16] d/control: bump versioned dependency for libpve-cluster-perl to have the new ssh related helpers available Signed-off-by: Thomas Lamprecht --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index d0b1d95..d7afa98 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,7 @@ Build-Depends: debhelper-compat (= 13), libfile-chdir-perl, libposix-strptime-perl, libpve-access-control, - libpve-cluster-perl, + libpve-cluster-perl (>= 8.0.6), libpve-common-perl (>= 6.3-2), librados2-perl, libtest-mockmodule-perl, @@ -35,7 +35,7 @@ Depends: ceph-common (>= 12.2~), libposix-strptime-perl, libpve-access-control (>= 8.1.2), libpve-apiclient-perl (>= 3.1-1), - libpve-cluster-perl, + libpve-cluster-perl (>= 8.0.6), libpve-common-perl (>= 8.1.1), librados2-perl, lvm2, -- 2.39.2 From a57207555baaf35c4391000be6c9c756e7e1fe60 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Fri, 19 Apr 2024 09:07:15 +0200 Subject: [PATCH 12/16] bump version to 8.2.0 Signed-off-by: Thomas Lamprecht --- debian/changelog | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/debian/changelog b/debian/changelog index aef5f84..22bf1c3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,16 @@ +libpve-storage-perl (8.2.0) bookworm; urgency=medium + + * fix insecure migration failing if it must wait on the lock + + * storage migrate: only output about cleaning-up snapshots if there are any + + * fix #1611: implement import of base-images for LVM-thin Storage + + * upload: use SSH helper to get ssh/scp options to ensure the new + node-specific public key files inside pmxcfs are used + + -- Proxmox Support Team Fri, 19 Apr 2024 09:07:08 +0200 + libpve-storage-perl (8.1.5) bookworm; urgency=medium * esxi: status: mark as active if its mounted -- 2.39.2 From b1d0effc9907beded6c8ee47a81317ff0f2d417f Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Tue, 23 Apr 2024 10:50:06 +0200 Subject: [PATCH 13/16] esxi import: add "longhorn" to possible Windows Server 2008 OS types While we detected "winLonghorn" already as w2k8, it seems that some ESXi versions also use "longhorn" here. Add that to our known Windows OS type mapping, so that it gets correctly mapped to our w2k8 type, which affects the selected hardware. Signed-off-by: Thomas Lamprecht --- src/PVE/Storage/ESXiPlugin.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PVE/Storage/ESXiPlugin.pm b/src/PVE/Storage/ESXiPlugin.pm index df00b73..b8bce0e 100644 --- a/src/PVE/Storage/ESXiPlugin.pm +++ b/src/PVE/Storage/ESXiPlugin.pm @@ -875,6 +875,7 @@ sub is_windows { my %guest_types_windows = ( 'dos' => 'other', + 'longhorn' => 'w2k8', 'winNetBusiness' => 'w2k3', 'windows9' => 'win10', 'windows9-64' => 'win10', -- 2.39.2 From bb7279806475f9326a7f934ab6c4f18a72a2e295 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Tue, 23 Apr 2024 17:20:47 +0200 Subject: [PATCH 14/16] bump version to 8.2.1 Signed-off-by: Thomas Lamprecht --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 22bf1c3..7eee922 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +libpve-storage-perl (8.2.1) bookworm; urgency=medium + + * esxi import: add "longhorn" to possible Windows Server 2008 OS types + + -- Proxmox Support Team Tue, 23 Apr 2024 17:20:10 +0200 + libpve-storage-perl (8.2.0) bookworm; urgency=medium * fix insecure migration failing if it must wait on the lock -- 2.39.2 From 22cb4b0b7812658ecc2444ca3ee51a04c50e4f57 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 30 Apr 2024 09:53:42 +0200 Subject: [PATCH 15/16] don't bail on whitespaces in backing devices This prevents importing from vmdks with whitespaces in file names. Further, some operations that include file sizes (like listing disks) would potentially fail entirely if a custom disk with a badly name backing device exists in a VM images directory since they don't expect this. Specifically, since we don't necessarily know the actual naming scheme of the current storage in the plain Plugin.pm version, we don't check the full name anyway, so why bother with whitespaces... See-also: https://forum.proxmox.com/threads/new-import-wizard-available-for-migrating-vmware-esxi-based-virtual-machines.144023/page-16#post-658697 Signed-off-by: Wolfgang Bumiller --- src/PVE/Storage/Plugin.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm index 22a9729..683190b 100644 --- a/src/PVE/Storage/Plugin.pm +++ b/src/PVE/Storage/Plugin.pm @@ -982,7 +982,7 @@ sub file_size_info { $used = int($used); ($format) = ($format =~ /^(\S+)$/) or die "format '$format' includes whitespace\n"; # untaint if (defined($parent)) { - ($parent) = ($parent =~ /^(\S+)$/) or die "parent '$parent' includes whitespace\n"; # untaint + ($parent) = ($parent =~ /^(\S+)$/); # untaint } return wantarray ? ($size, $format, $used, $parent, $st->ctime) : $size; } -- 2.39.2 From 98a5bc5c9b5a030cef88629900cc1214120feb61 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 30 Apr 2024 09:53:43 +0200 Subject: [PATCH 16/16] fixup error messages when getting file size info The assignment happens before the 'die', so the error message would always contain 'undef'. Signed-off-by: Wolfgang Bumiller --- src/PVE/Storage/Plugin.pm | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm index 683190b..b5a54c1 100644 --- a/src/PVE/Storage/Plugin.pm +++ b/src/PVE/Storage/Plugin.pm @@ -974,13 +974,16 @@ sub file_size_info { my ($size, $format, $used, $parent) = $info->@{qw(virtual-size format actual-size backing-filename)}; - ($size) = ($size =~ /^(\d+)$/) or die "size '$size' not an integer\n"; # untaint + ($size) = ($size =~ /^(\d+)$/); # untaint + die "size '$size' not an integer\n" if !defined($size); # coerce back from string $size = int($size); - ($used) = ($used =~ /^(\d+)$/) or die "used '$used' not an integer\n"; # untaint + ($used) = ($used =~ /^(\d+)$/); # untaint + die "used '$used' not an integer\n" if !defined($used); # coerce back from string $used = int($used); - ($format) = ($format =~ /^(\S+)$/) or die "format '$format' includes whitespace\n"; # untaint + ($format) = ($format =~ /^(\S+)$/); # untaint + die "format '$format' includes whitespace\n" if !defined($format); if (defined($parent)) { ($parent) = ($parent =~ /^(\S+)$/); # untaint } -- 2.39.2