use JSON;
use IO::File;
use String::ShellQuote 'shell_quote';
+use Text::ParseWords;
my $PROGNAME = "pve-zsync";
my $CONFIG_PATH = "/var/lib/${PROGNAME}";
# targets are either a VMID, or a 'host:zpool/path' with 'host:' being optional
my $TARGETRE = qr!^(?:($HOSTRE):)?(\d+|(?:[\w\-_]+)(/.+)?)$!;
-my $DISK_KEY_RE = qr/^(?:(?:(?:virtio|ide|scsi|sata|efidisk|mp)\d+)|rootfs): /;
+my $DISK_KEY_RE = qr/^(?:(?:(?:virtio|ide|scsi|sata|efidisk|tpmstate|mp)\d+)|rootfs): /;
my $INSTANCE_ID = get_instance_id($$);
my $text = read_file($CRONJOBS, 0);
- return encode_cron(@{$text});
+ return parse_cron(@{$text});
}
sub parse_argv {
verbose => undef,
limit => undef,
maxsnap => undef,
+ dest_maxsnap => undef,
name => undef,
skip => undef,
method => undef,
source_user => undef,
dest_user => undef,
prepend_storage_id => undef,
+ compressed => undef,
properties => undef,
dest_config_path => undef,
};
'verbose' => \$param->{verbose},
'limit=i' => \$param->{limit},
'maxsnap=i' => \$param->{maxsnap},
+ 'dest-maxsnap=i' => \$param->{dest_maxsnap},
'name=s' => \$param->{name},
'skip' => \$param->{skip},
'method=s' => \$param->{method},
'source-user=s' => \$param->{source_user},
'dest-user=s' => \$param->{dest_user},
'prepend-storage-id' => \$param->{prepend_storage_id},
+ 'compressed' => \$param->{compressed},
'properties' => \$param->{properties},
'dest-config-path=s' => \$param->{dest_config_path},
);
return $job;
}
-sub encode_cron {
+sub parse_cron {
my (@text) = @_;
my $cfg = {};
while (my $line = shift(@text)) {
-
- my @arg = split('\s', $line);
+ my @arg = Text::ParseWords::shellwords($line);
my $param = parse_argv(@arg);
if ($param->{source} && $param->{dest}) {
$job->{method} = "local" if !$dest->{ip} && !$source->{ip};
$job->{method} = "ssh" if !$job->{method};
$job->{limit} = $param->{limit};
- $job->{maxsnap} = $param->{maxsnap} if $param->{maxsnap};
+ $job->{maxsnap} = $param->{maxsnap};
+ $job->{dest_maxsnap} = $param->{dest_maxsnap};
$job->{source} = $param->{source};
$job->{source_user} = $param->{source_user};
$job->{dest_user} = $param->{dest_user};
$job->{prepend_storage_id} = !!$param->{prepend_storage_id};
+ $job->{compressed} = !!$param->{compressed};
$job->{properties} = !!$param->{properties};
$job->{dest_config_path} = $param->{dest_config_path} if $param->{dest_config_path};
$text .= " root";
$text .= " $PROGNAME sync --source $job->{source} --dest $job->{dest}";
$text .= " --name $job->{name} --maxsnap $job->{maxsnap}";
+ $text .= " --dest-maxsnap $job->{dest_maxsnap}" if defined($job->{dest_maxsnap});
$text .= " --limit $job->{limit}" if $job->{limit};
$text .= " --method $job->{method}";
$text .= " --verbose" if $job->{verbose};
$text .= " --source-user $job->{source_user}";
$text .= " --dest-user $job->{dest_user}";
$text .= " --prepend-storage-id" if $job->{prepend_storage_id};
+ $text .= " --compressed" if $job->{compressed};
$text .= " --properties" if $job->{properties};
$text .= " --dest-config-path $job->{dest_config_path}" if $job->{dest_config_path};
$text .= "\n";
my $sync_path = sub {
my ($source, $dest, $job, $param, $date) = @_;
- ($dest->{old_snap}, $dest->{last_snap}) = snapshot_get($source, $dest, $param->{maxsnap}, $param->{name}, $param->{dest_user});
+ my $dest_dataset = target_dataset($source, $dest);
+
+ ($dest->{old_snap}, $dest->{last_snap}) = snapshot_get(
+ $dest_dataset,
+ $param->{dest_maxsnap} // $param->{maxsnap},
+ $param->{name},
+ $dest->{ip},
+ $param->{dest_user},
+ );
+
+ ($source->{old_snap}) = snapshot_get(
+ $source->{all},
+ $param->{maxsnap},
+ $param->{name},
+ $source->{ip},
+ $param->{source_user},
+ );
prepare_prepended_target($source, $dest, $param->{dest_user}) if defined($dest->{prepend});
send_image($source, $dest, $param);
- snapshot_destroy($source, $dest, $param->{method}, $dest->{old_snap}, $param->{source_user}, $param->{dest_user}) if ($source->{destroy} && $dest->{old_snap});
+ for my $old_snap (@{$source->{old_snap}}) {
+ snapshot_destroy($source->{all}, $old_snap, $source->{ip}, $param->{source_user});
+ }
+ for my $old_snap (@{$dest->{old_snap}}) {
+ snapshot_destroy($dest_dataset, $old_snap, $dest->{ip}, $param->{dest_user});
+ }
};
eval{
}
sub snapshot_get{
- my ($source, $dest, $max_snap, $name, $dest_user) = @_;
+ my ($dataset, $max_snap, $name, $ip, $user) = @_;
my $cmd = [];
- push @$cmd, 'ssh', "$dest_user\@$dest->{ip}", '--', if $dest->{ip};
+ push @$cmd, 'ssh', "$user\@$ip", '--', if $ip;
push @$cmd, 'zfs', 'list', '-r', '-t', 'snapshot', '-Ho', 'name', '-S', 'creation';
-
- my $path = target_dataset($source, $dest);
- push @$cmd, $path;
+ push @$cmd, $dataset;
my $raw;
eval {$raw = run_cmd($cmd)};
my $index = 0;
my $line = "";
my $last_snap = undef;
- my $old_snap;
+ my $old_snap = [];
while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
$line = $1;
$last_snap = $1 if (!$last_snap);
}
if ($line =~ m/(rep_\Q${name}\E_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})$/) {
- $old_snap = $1;
+ # interpreted as infinity
+ last if $max_snap <= 0;
+
+ my $snap = $1;
$index++;
- if ($index == $max_snap) {
- $source->{destroy} = 1;
- last;
- };
+
+ if ($index >= $max_snap) {
+ push @{$old_snap}, $snap;
+ }
}
}
};
if (my $err = $@) {
- snapshot_destroy($source, $dest, 'ssh', $snap_name, $source_user, $dest_user);
+ snapshot_destroy($source->{all}, $snap_name, $source->{ip}, $source_user);
die "$err\n";
}
}
$num++;
} else {
- die "ERROR: in path\n";
+ die "unexpected path '$path'\n";
}
}
- die "Vm include no disk on zfs.\n" if !$disks->{0};
+ die "Guest does not include any ZFS volumes (or all are excluded by the backup flag).\n"
+ if !$disks->{0};
return $disks;
}
}
sub snapshot_destroy {
- my ($source, $dest, $method, $snap, $source_user, $dest_user) = @_;
+ my ($dataset, $snap, $ip, $user) = @_;
my @zfscmd = ('zfs', 'destroy');
- my $snapshot = "$source->{all}\@$snap";
+ my $snapshot = "$dataset\@$snap";
eval {
- if($source->{ip} && $method eq 'ssh'){
- run_cmd(['ssh', "$source_user\@$source->{ip}", '--', @zfscmd, $snapshot]);
+ if ($ip) {
+ run_cmd(['ssh', "$user\@$ip", '--', @zfscmd, $snapshot]);
} else {
run_cmd([@zfscmd, $snapshot]);
}
if (my $erro = $@) {
warn "WARN: $erro";
}
- if ($dest) {
- my @ssh = $dest->{ip} ? ('ssh', "$dest_user\@$dest->{ip}", '--') : ();
-
- my $path = target_dataset($source, $dest);
-
- eval {
- run_cmd([@ssh, @zfscmd, "$path\@$snap"]);
- };
- if (my $erro = $@) {
- warn "WARN: $erro";
- }
- }
}
# check if snapshot for incremental sync exist on source side
push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{source_user}\@$source->{ip}", '--' if $source->{ip};
push @$cmd, 'zfs', 'send';
+ push @$cmd, '-L', if $param->{compressed}; # no effect if dataset never had large recordsize
+ push @$cmd, '-c', if $param->{compressed};
push @$cmd, '-p', if $param->{properties};
push @$cmd, '-v' if $param->{verbose};
};
if (my $erro = $@) {
- snapshot_destroy($source, undef, $param->{method}, $source->{new_snap}, $param->{source_user}, $param->{dest_user});
+ snapshot_destroy($source->{all}, $source->{new_snap}, $source->{ip}, $param->{source_user});
die $erro;
};
}
run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", $dest_target_new]);
}
- if ($source->{destroy}){
- my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.$dest->{old_snap}";
+ for my $old_snap (@{$dest->{old_snap}}) {
+ my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.${old_snap}";
if($dest->{ip}){
run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'rm', '-f', '--', $dest_target_old]);
} else {
Maximal sync speed in kBytes/s, default is unlimited
--maxsnap integer
- How much snapshots will be kept before get erased, default 1
+ The number of snapshots to keep until older ones are erased.
+ The default is 1, use 0 for unlimited.
+
+ --dest-maxsnap integer
+ Override maxsnap for the destination dataset.
--name string
The name of the sync job, if not set it is default
--source-user string
The (ssh) user-name on the source target, root by default
+ --compressed
+ If specified, send data without decompressing first. If features lz4_compress,
+ zstd_compress or large_blocks are in use by the source, they need to be enabled on
+ the target as well.
+
--properties
If specified, include the dataset's properties in the stream.
The maximal sync speed in kBytes/s, default is unlimited
--maxsnap integer
- Configure how many snapshots will be kept before get erased, default 1
+ The number of snapshots to keep until older ones are erased.
+ The default is 1, use 0 for unlimited.
+
+ --dest-maxsnap integer
+ Override maxsnap for the destination dataset.
--name string
The name of the sync job, if not set it is 'default'.
--verbose
If specified, print out the sync progress.
+ --compressed
+ If specified, send data without decompressing first. If features lz4_compress,
+ zstd_compress or large_blocks are in use by the source, they need to be enabled on
+ the target as well.
+
--properties
If specified, include the dataset's properties in the stream.
sub print_pod {
my $synopsis = join("\n", sort values %$cmd_help);
+ my $commands = join(", ", sort keys %$cmd_help);
print <<EOF;
=head1 NAME
-pve-zsync - PVE ZFS Replication Manager
+pve-zsync - PVE ZFS Storage Sync Tool
=head1 SYNOPSIS
pve-zsync <COMMAND> [ARGS] [OPTIONS]
-$synopsis
+Where <COMMAND> can be one of: $commands
=head1 DESCRIPTION
-This Tool helps you to sync your VM or directory which stored on ZFS between 2 servers.
-This tool also has the capability to add jobs to cron so the sync will be automatically done.
-The default syncing interval is set to 15 min, if you want to change this value you can do this in /etc/cron.d/pve-zsync.
-To config cron see man crontab.
+The pve-zsync tool can help you to sync your VMs or directories stored on ZFS
+between multiple servers.
-=head2 PVE ZFS Storage sync Tool
+pve-zsync is able to automatically configure CRON jobs, so that a periodic sync
+will be automatically triggered.
+The default sync interval is 15 min, if you want to change this value you can
+do this in F</etc/cron.d/pve-zsync>. If you need help to configure CRON tabs, see
+man crontab.
-This Tool can get remote pool on other PVE or send Pool to others ZFS machines
+=head1 COMMANDS AND OPTIONS
+
+$synopsis
=head1 EXAMPLES
-add sync job from local VM to remote ZFS Server
-pve-zsync create -source=100 -dest=192.168.1.2:zfspool
+Adds a job for syncing the local VM 100 to a remote server's ZFS pool named "tank":
+ pve-zsync create --source=100 -dest=192.168.1.2:tank
=head1 IMPORTANT FILES
-Cron jobs and config are stored at /etc/cron.d/pve-zsync
+Cron jobs and config are stored in F</etc/cron.d/pve-zsync>
-The VM config get copied on the destination machine to /var/lib/pve-zsync/
+The VM configuration itself gets copied to the destination machines
+F</var/lib/pve-zsync/> path.
=head1 COPYRIGHT AND DISCLAIMER
Copyright (C) 2007-2021 Proxmox Server Solutions GmbH
-This program is free software: you can redistribute it and/or modify it
-under the terms of the GNU Affero General Public License as published
-by the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU Affero General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.
-This program is distributed in the hope that it will be useful, but
-WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-Affero General Public License for more details.
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+details.
-You should have received a copy of the GNU Affero General Public
-License along with this program. If not, see
-<http://www.gnu.org/licenses/>.
+You should have received a copy of the GNU Affero General Public License along
+with this program. If not, see <http://www.gnu.org/licenses/>.
EOF
}
projects / pve-zsync.git / blobdiff