bump version to 2.3.1

[pve-zsync.git] / pve-zsync

diff --git a/pve-zsync b/pve-zsync
index d14a1c165a998b333c91f849bbff14e189e0249d..de5d46f529f6a6f897d56349961fda11d60c74d3 100755 (executable)
--- a/pve-zsync
+++ b/pve-zsync
@@ -9,6 +9,7 @@ use File::Path qw(make_path);
 use JSON;
 use IO::File;
 use String::ShellQuote 'shell_quote';
 use JSON;
 use IO::File;
 use String::ShellQuote 'shell_quote';

+use Text::ParseWords;

 
 my $PROGNAME = "pve-zsync";
 my $CONFIG_PATH = "/var/lib/${PROGNAME}";
 
 my $PROGNAME = "pve-zsync";
 my $CONFIG_PATH = "/var/lib/${PROGNAME}";


@@ -53,6 +54,10 @@ my $HOSTRE = "(?:$HOSTv4RE1|\\[$IPV6RE\\])";       # ipv6 must always be in brac
 # targets are either a VMID, or a 'host:zpool/path' with 'host:' being optional
 my $TARGETRE = qr!^(?:($HOSTRE):)?(\d+|(?:[\w\-_]+)(/.+)?)$!;
 
 # targets are either a VMID, or a 'host:zpool/path' with 'host:' being optional
 my $TARGETRE = qr!^(?:($HOSTRE):)?(\d+|(?:[\w\-_]+)(/.+)?)$!;
 

+my $DISK_KEY_RE = qr/^(?:(?:(?:virtio|ide|scsi|sata|efidisk|tpmstate|mp)\d+)|rootfs): /;
+
+my $INSTANCE_ID = get_instance_id($$);
+

 my $command = $ARGV[0];
 
 if (defined($command) && $command ne 'help' && $command ne 'printpod') {
 my $command = $ARGV[0];
 
 if (defined($command) && $command ne 'help' && $command ne 'printpod') {


@@ -79,6 +84,19 @@ sub check_bin {
     die "unable to find command '$bin'\n";
 }
 
     die "unable to find command '$bin'\n";
 }
 

+sub read_file {
+    my ($filename, $one_line_only) = @_;
+
+    my $fh = IO::File->new($filename, "r")
+       or die "Could not open file ${filename}: $!\n";
+
+    my $text = $one_line_only ? <$fh> : [ <$fh> ];
+
+    close($fh);
+
+    return $text;
+}
+

 sub cut_target_width {
     my ($path, $maxlen) = @_;
     $path =~ s@/+@/@g;
 sub cut_target_width {
     my ($path, $maxlen) = @_;
     $path =~ s@/+@/@g;


@@ -135,15 +153,15 @@ sub get_status {
     return undef;
 }
 
     return undef;
 }
 

-sub check_pool_exists {
-    my ($target, $user) = @_;
+sub check_dataset_exists {
+    my ($dataset, $ip, $user) = @_;

 
     my $cmd = [];
 
 
     my $cmd = [];
 

-    if ($target->{ip}) {
-       push @$cmd, 'ssh', "$user\@$target->{ip}", '--';
+    if ($ip) {
+       push @$cmd, 'ssh', "$user\@$ip", '--';

     }
     }

-    push @$cmd, 'zfs', 'list', '-H', '--', $target->{all};
+    push @$cmd, 'zfs', 'list', '-H', '--', $dataset;

     eval {
        run_cmd($cmd);
     };
     eval {
        run_cmd($cmd);
     };


@@ -154,6 +172,19 @@ sub check_pool_exists {
     return 1;
 }
 
     return 1;
 }
 

+sub create_file_system {
+    my ($file_system, $ip, $user) = @_;
+
+    my $cmd = [];
+
+    if ($ip) {
+       push @$cmd, 'ssh', "$user\@$ip", '--';
+    }
+    push @$cmd, 'zfs', 'create', $file_system;
+
+    run_cmd($cmd);
+}
+

 sub parse_target {
     my ($text) = @_;
 
 sub parse_target {
     my ($text) = @_;
 


@@ -200,14 +231,9 @@ sub read_cron {
        return undef;
     }
 
        return undef;
     }
 

-    my $fh = IO::File->new("< $CRONJOBS");
-    die "Could not open file $CRONJOBS: $!\n" if !$fh;
-
-    my @text = <$fh>;
+    my $text = read_file($CRONJOBS, 0);

 
 

-    close($fh);
-
-    return encode_cron(@text);
+    return parse_cron(@{$text});

 }
 
 sub parse_argv {
 }
 
 sub parse_argv {


@@ -219,11 +245,14 @@ sub parse_argv {
        verbose => undef,
        limit => undef,
        maxsnap => undef,
        verbose => undef,
        limit => undef,
        maxsnap => undef,

+       dest_maxsnap => undef,

        name => undef,
        skip => undef,
        method => undef,
        source_user => undef,
        dest_user => undef,
        name => undef,
        skip => undef,
        method => undef,
        source_user => undef,
        dest_user => undef,

+       prepend_storage_id => undef,
+       compressed => undef,

        properties => undef,
        dest_config_path => undef,
     };
        properties => undef,
        dest_config_path => undef,
     };


@@ -235,11 +264,14 @@ sub parse_argv {
        'verbose' => \$param->{verbose},
        'limit=i' => \$param->{limit},
        'maxsnap=i' => \$param->{maxsnap},
        'verbose' => \$param->{verbose},
        'limit=i' => \$param->{limit},
        'maxsnap=i' => \$param->{maxsnap},

+       'dest-maxsnap=i' => \$param->{dest_maxsnap},

        'name=s' => \$param->{name},
        'skip' => \$param->{skip},
        'method=s' => \$param->{method},
        'source-user=s' => \$param->{source_user},
        'dest-user=s' => \$param->{dest_user},
        'name=s' => \$param->{name},
        'skip' => \$param->{skip},
        'method=s' => \$param->{method},
        'source-user=s' => \$param->{source_user},
        'dest-user=s' => \$param->{dest_user},

+       'prepend-storage-id' => \$param->{prepend_storage_id},
+       'compressed' => \$param->{compressed},

        'properties' => \$param->{properties},
        'dest-config-path=s' => \$param->{dest_config_path},
     );
        'properties' => \$param->{properties},
        'dest-config-path=s' => \$param->{dest_config_path},
     );


@@ -264,6 +296,7 @@ sub add_state_to_job {
     $job->{state} = $state->{state};
     $job->{lsync} = $state->{lsync};
     $job->{vm_type} = $state->{vm_type};
     $job->{state} = $state->{state};
     $job->{lsync} = $state->{lsync};
     $job->{vm_type} = $state->{vm_type};

+    $job->{instance_id} = $state->{instance_id};

 
     for (my $i = 0; $state->{"snap$i"}; $i++) {
        $job->{"snap$i"} = $state->{"snap$i"};
 
     for (my $i = 0; $state->{"snap$i"}; $i++) {
        $job->{"snap$i"} = $state->{"snap$i"};


@@ -272,14 +305,13 @@ sub add_state_to_job {
     return $job;
 }
 
     return $job;
 }
 

-sub encode_cron {
+sub parse_cron {

     my (@text) = @_;
 
     my $cfg = {};
 
     while (my $line = shift(@text)) {
     my (@text) = @_;
 
     my $cfg = {};
 
     while (my $line = shift(@text)) {

-
-       my @arg = split('\s', $line);
+       my @arg = Text::ParseWords::shellwords($line);

        my $param = parse_argv(@arg);
 
        if ($param->{source} && $param->{dest}) {
        my $param = parse_argv(@arg);
 
        if ($param->{source} && $param->{dest}) {


@@ -299,17 +331,21 @@ sub param_to_job {
     my $job = {};
 
     my $source = parse_target($param->{source});
     my $job = {};
 
     my $source = parse_target($param->{source});

-    my $dest = parse_target($param->{dest}) if $param->{dest};
+    my $dest;
+    $dest = parse_target($param->{dest}) if $param->{dest};

 
     $job->{name} = !$param->{name} ? "default" : $param->{name};
     $job->{dest} = $param->{dest} if $param->{dest};
     $job->{method} = "local" if !$dest->{ip} && !$source->{ip};
     $job->{method} = "ssh" if !$job->{method};
     $job->{limit} = $param->{limit};
 
     $job->{name} = !$param->{name} ? "default" : $param->{name};
     $job->{dest} = $param->{dest} if $param->{dest};
     $job->{method} = "local" if !$dest->{ip} && !$source->{ip};
     $job->{method} = "ssh" if !$job->{method};
     $job->{limit} = $param->{limit};

-    $job->{maxsnap} = $param->{maxsnap} if $param->{maxsnap};
+    $job->{maxsnap} = $param->{maxsnap};
+    $job->{dest_maxsnap} = $param->{dest_maxsnap};

     $job->{source} = $param->{source};
     $job->{source_user} = $param->{source_user};
     $job->{dest_user} = $param->{dest_user};
     $job->{source} = $param->{source};
     $job->{source_user} = $param->{source_user};
     $job->{dest_user} = $param->{dest_user};

+    $job->{prepend_storage_id} = !!$param->{prepend_storage_id};
+    $job->{compressed} = !!$param->{compressed};

     $job->{properties} = !!$param->{properties};
     $job->{dest_config_path} = $param->{dest_config_path} if $param->{dest_config_path};
 
     $job->{properties} = !!$param->{properties};
     $job->{dest_config_path} = $param->{dest_config_path} if $param->{dest_config_path};
 


@@ -327,28 +363,14 @@ sub read_state {
        return undef;
     }
 
        return undef;
     }
 

-    my $fh = IO::File->new("< $STATE");
-    die "Could not open file $STATE: $!\n" if !$fh;
-
-    my $text = <$fh>;
-    my $states = decode_json($text);
-
-    close($fh);
-
-    return $states;
+    my $text = read_file($STATE, 1);
+    return decode_json($text);

 }
 
 sub update_state {
     my ($job) = @_;
 }
 
 sub update_state {
     my ($job) = @_;

-    my $text;
-    my $in_fh;
-
-    eval {

 
 

-       $in_fh = IO::File->new("< $STATE");
-       die "Could not open file $STATE: $!\n" if !$in_fh;
-       $text = <$in_fh>;
-    };
+    my $text = eval { read_file($STATE, 1); };

 
     my $out_fh = IO::File->new("> $STATE.new");
     die "Could not open file ${STATE}.new: $!\n" if !$out_fh;
 
     my $out_fh = IO::File->new("> $STATE.new");
     die "Could not open file ${STATE}.new: $!\n" if !$out_fh;


@@ -363,6 +385,7 @@ sub update_state {
     if ($job->{state} ne "del") {
        $state->{state} = $job->{state};
        $state->{lsync} = $job->{lsync};
     if ($job->{state} ne "del") {
        $state->{state} = $job->{state};
        $state->{lsync} = $job->{lsync};

+       $state->{instance_id} = $job->{instance_id};

        $state->{vm_type} = $job->{vm_type};
 
        for (my $i = 0; $job->{"snap$i"} ; $i++) {
        $state->{vm_type} = $job->{vm_type};
 
        for (my $i = 0; $job->{"snap$i"} ; $i++) {


@@ -380,9 +403,6 @@ sub update_state {
 
     close($out_fh);
     rename "$STATE.new", $STATE;
 
     close($out_fh);
     rename "$STATE.new", $STATE;

-    eval {
-       close($in_fh);
-    };

 
     return $states;
 }
 
     return $states;
 }


@@ -397,12 +417,9 @@ sub update_cron {
     my $header = "SHELL=/bin/sh\n";
     $header .= "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n\n";
 
     my $header = "SHELL=/bin/sh\n";
     $header .= "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n\n";
 

-    my $fh = IO::File->new("< $CRONJOBS");
-    die "Could not open file $CRONJOBS: $!\n" if !$fh;
+    my $current = read_file($CRONJOBS, 0);

 
 

-    my @test = <$fh>;
-
-    while (my $line = shift(@test)) {
+    foreach my $line (@{$current}) {

        chomp($line);
        if ($line =~ m/source $job->{source} .*name $job->{name} /) {
            $updated = 1;
        chomp($line);
        if ($line =~ m/source $job->{source} .*name $job->{name} /) {
            $updated = 1;


@@ -427,11 +444,10 @@ sub update_cron {
     my $new_fh = IO::File->new("> ${CRONJOBS}.new");
     die "Could not open file ${CRONJOBS}.new: $!\n" if !$new_fh;
 
     my $new_fh = IO::File->new("> ${CRONJOBS}.new");
     die "Could not open file ${CRONJOBS}.new: $!\n" if !$new_fh;
 

-    die "can't write to $CRONJOBS.new\n" if !print($new_fh $text);
+    print $new_fh $text or die "can't write to $CRONJOBS.new: $!\n";

     close ($new_fh);
 
     close ($new_fh);
 

-    die "can't move $CRONJOBS.new: $!\n" if !rename "${CRONJOBS}.new", $CRONJOBS;
-    close ($fh);
+    rename "${CRONJOBS}.new", $CRONJOBS or die "can't move $CRONJOBS.new: $!\n";

 }
 
 sub format_job {
 }
 
 sub format_job {


@@ -450,11 +466,14 @@ sub format_job {
     $text .= " root";
     $text .= " $PROGNAME sync --source $job->{source} --dest $job->{dest}";
     $text .= " --name $job->{name} --maxsnap $job->{maxsnap}";
     $text .= " root";
     $text .= " $PROGNAME sync --source $job->{source} --dest $job->{dest}";
     $text .= " --name $job->{name} --maxsnap $job->{maxsnap}";

+    $text .= " --dest-maxsnap $job->{dest_maxsnap}" if defined($job->{dest_maxsnap});

     $text .= " --limit $job->{limit}" if $job->{limit};
     $text .= " --method $job->{method}";
     $text .= " --verbose" if $job->{verbose};
     $text .= " --source-user $job->{source_user}";
     $text .= " --dest-user $job->{dest_user}";
     $text .= " --limit $job->{limit}" if $job->{limit};
     $text .= " --method $job->{method}";
     $text .= " --verbose" if $job->{verbose};
     $text .= " --source-user $job->{source_user}";
     $text .= " --dest-user $job->{dest_user}";

+    $text .= " --prepend-storage-id" if $job->{prepend_storage_id};
+    $text .= " --compressed" if $job->{compressed};

     $text .= " --properties" if $job->{properties};
     $text .= " --dest-config-path $job->{dest_config_path}" if $job->{dest_config_path};
     $text .= "\n";
     $text .= " --properties" if $job->{properties};
     $text .= " --dest-config-path $job->{dest_config_path}" if $job->{dest_config_path};
     $text .= "\n";


@@ -524,10 +543,12 @@ sub init {
            run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{source_user}\@$ip"]);
        }
 
            run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{source_user}\@$ip"]);
        }
 

-       die "Pool $dest->{all} does not exists\n" if !check_pool_exists($dest, $param->{dest_user});
+       die "Pool $dest->{all} does not exist\n"
+           if !check_dataset_exists($dest->{all}, $dest->{ip}, $param->{dest_user});

 
        if (!defined($source->{vmid})) {
 
        if (!defined($source->{vmid})) {

-           die "Pool $source->{all} does not exists\n" if !check_pool_exists($source, $param->{source_user});
+           die "Pool $source->{all} does not exist\n"
+               if !check_dataset_exists($source->{all}, $source->{ip}, $param->{source_user});

        }
 
        my $vm_type = vm_exists($source, $param->{source_user});
        }
 
        my $vm_type = vm_exists($source, $param->{source_user});


@@ -545,10 +566,10 @@ sub init {
        update_state($job);
     }); #cron and state lock
 
        update_state($job);
     }); #cron and state lock
 

-    eval {
-       sync($param) if !$param->{skip};
-    };
-    if(my $err = $@) {
+    return if $param->{skip};
+
+    eval { sync($param) };
+    if (my $err = $@) {

        destroy_job($param);
        print $err;
     }
        destroy_job($param);
        print $err;
     }


@@ -582,6 +603,33 @@ sub destroy_job {
     });
 }
 
     });
 }
 

+sub get_instance_id {
+    my ($pid) = @_;
+
+    my $stat = read_file("/proc/$pid/stat", 1)
+       or die "unable to read process stats\n";
+    my $boot_id = read_file("/proc/sys/kernel/random/boot_id", 1)
+       or die "unable to read boot ID\n";
+
+    my $stats = [ split(/\s+/, $stat) ];
+    my $starttime = $stats->[21];
+    chomp($boot_id);
+
+    return "${pid}:${starttime}:${boot_id}";
+}
+
+sub instance_exists {
+    my ($instance_id) = @_;
+
+    if (defined($instance_id) && $instance_id =~ m/^([1-9][0-9]*):/) {
+       my $pid = $1;
+       my $actual_id = eval { get_instance_id($pid); };
+       return defined($actual_id) && $actual_id eq $instance_id;
+    }
+
+    return 0;
+}
+

 sub sync {
     my ($param) = @_;
 
 sub sync {
     my ($param) = @_;
 


@@ -591,11 +639,16 @@ sub sync {
        eval { $job = get_job($param) };
 
        if ($job) {
        eval { $job = get_job($param) };
 
        if ($job) {

-           if (defined($job->{state}) && ($job->{state} eq "syncing" || $job->{state} eq "waiting")) {
+           my $state = $job->{state} // 'ok';
+           $state = 'ok' if !instance_exists($job->{instance_id});
+
+           if ($state eq "syncing" || $state eq "waiting") {

                die "Job --source $param->{source} --name $param->{name} is already scheduled to sync\n";
            }
 
            $job->{state} = "waiting";
                die "Job --source $param->{source} --name $param->{name} is already scheduled to sync\n";
            }
 
            $job->{state} = "waiting";

+           $job->{instance_id} = $INSTANCE_ID;
+

            update_state($job);
        }
     });
            update_state($job);
        }
     });


@@ -632,14 +685,37 @@ sub sync {
        my $sync_path = sub {
            my ($source, $dest, $job, $param, $date) = @_;
 
        my $sync_path = sub {
            my ($source, $dest, $job, $param, $date) = @_;
 

-           ($source->{old_snap}, $source->{last_snap}) = snapshot_get($source, $dest, $param->{maxsnap}, $param->{name}, $param->{source_user});
+           my $dest_dataset = target_dataset($source, $dest);
+
+           ($dest->{old_snap}, $dest->{last_snap}) = snapshot_get(
+               $dest_dataset,
+               $param->{dest_maxsnap} // $param->{maxsnap},
+               $param->{name},
+               $dest->{ip},
+               $param->{dest_user},
+           );
+
+           ($source->{old_snap}) = snapshot_get(
+               $source->{all},
+               $param->{maxsnap},
+               $param->{name},
+               $source->{ip},
+               $param->{source_user},
+           );
+
+           prepare_prepended_target($source, $dest, $param->{dest_user}) if defined($dest->{prepend});

 
            snapshot_add($source, $dest, $param->{name}, $date, $param->{source_user}, $param->{dest_user});
 
            send_image($source, $dest, $param);
 
 
            snapshot_add($source, $dest, $param->{name}, $date, $param->{source_user}, $param->{dest_user});
 
            send_image($source, $dest, $param);
 

-           snapshot_destroy($source, $dest, $param->{method}, $source->{old_snap}, $param->{source_user}, $param->{dest_user}) if ($source->{destroy} && $source->{old_snap});
+           for my $old_snap (@{$source->{old_snap}}) {
+               snapshot_destroy($source->{all}, $old_snap, $source->{ip}, $param->{source_user});
+           }

 
 

+           for my $old_snap (@{$dest->{old_snap}}) {
+               snapshot_destroy($dest_dataset, $old_snap, $dest->{ip}, $param->{dest_user});
+           }

        };
 
        eval{
        };
 
        eval{


@@ -653,6 +729,10 @@ sub sync {
                    $source->{pool} = $disks->{$disk}->{pool};
                    $source->{path} = $disks->{$disk}->{path} if $disks->{$disk}->{path};
                    $source->{last_part} = $disks->{$disk}->{last_part};
                    $source->{pool} = $disks->{$disk}->{pool};
                    $source->{path} = $disks->{$disk}->{path} if $disks->{$disk}->{path};
                    $source->{last_part} = $disks->{$disk}->{last_part};

+
+                   $dest->{prepend} = $disks->{$disk}->{storage_id}
+                       if $param->{prepend_storage_id};
+

                    &$sync_path($source, $dest, $job, $param, $date);
                }
                if ($param->{method} eq "ssh" && ($source->{ip} || $dest->{ip})) {
                    &$sync_path($source, $dest, $job, $param, $date);
                }
                if ($param->{method} eq "ssh" && ($source->{ip} || $dest->{ip})) {


@@ -669,6 +749,7 @@ sub sync {
                eval { $job = get_job($param); };
                if ($job) {
                    $job->{state} = "error";
                eval { $job = get_job($param); };
                if ($job) {
                    $job->{state} = "error";

+                   delete $job->{instance_id};

                    update_state($job);
                }
            });
                    update_state($job);
                }
            });


@@ -685,6 +766,7 @@ sub sync {
                    $job->{state} = "ok";
                }
                $job->{lsync} = $date;
                    $job->{state} = "ok";
                }
                $job->{lsync} = $date;

+               delete $job->{instance_id};

                update_state($job);
            }
        });
                update_state($job);
            }
        });


@@ -692,30 +774,39 @@ sub sync {
 }
 
 sub snapshot_get{
 }
 
 sub snapshot_get{

-    my ($source, $dest, $max_snap, $name, $source_user) = @_;
+    my ($dataset, $max_snap, $name, $ip, $user) = @_;

 
     my $cmd = [];
 
     my $cmd = [];

-    push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--', if $source->{ip};
+    push @$cmd, 'ssh', "$user\@$ip", '--', if $ip;

     push @$cmd, 'zfs', 'list', '-r', '-t', 'snapshot', '-Ho', 'name', '-S', 'creation';
     push @$cmd, 'zfs', 'list', '-r', '-t', 'snapshot', '-Ho', 'name', '-S', 'creation';

-    push @$cmd, $source->{all};
+    push @$cmd, $dataset;
+
+    my $raw;
+    eval {$raw = run_cmd($cmd)};
+    if (my $erro =$@) { #this means the volume doesn't exist on dest yet
+       return undef;
+    }

 
 

-    my $raw = run_cmd($cmd);

     my $index = 0;
     my $line = "";
     my $last_snap = undef;
     my $index = 0;
     my $line = "";
     my $last_snap = undef;

-    my $old_snap;
+    my $old_snap = [];

 
     while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
        $line = $1;
 
     while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
        $line = $1;

+       if ($line =~ m/@(.*)$/) {
+           $last_snap = $1 if (!$last_snap);
+       }

        if ($line =~ m/(rep_\Q${name}\E_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})$/) {
        if ($line =~ m/(rep_\Q${name}\E_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})$/) {

+           # interpreted as infinity
+           last if $max_snap <= 0;

 
 

-           $last_snap = $1 if (!$last_snap);
-           $old_snap = $1;
+           my $snap = $1;

            $index++;
            $index++;

-           if ($index == $max_snap) {
-               $source->{destroy} = 1;
-               last;
-           };
+
+           if ($index >= $max_snap) {
+               push @{$old_snap}, $snap;
+           }

        }
     }
 
        }
     }
 


@@ -741,47 +832,11 @@ sub snapshot_add {
     };
 
     if (my $err = $@) {
     };
 
     if (my $err = $@) {

-       snapshot_destroy($source, $dest, 'ssh', $snap_name, $source_user, $dest_user);
+       snapshot_destroy($source->{all}, $snap_name, $source->{ip}, $source_user);

        die "$err\n";
     }
 }
 
        die "$err\n";
     }
 }
 

-sub write_cron {
-    my ($cfg) = @_;
-
-    my $text = "SHELL=/bin/sh\n";
-    $text .= "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n";
-
-    my $fh = IO::File->new("> $CRONJOBS");
-    die "Could not open file: $!\n" if !$fh;
-
-    foreach my $source (sort keys%{$cfg}) {
-       foreach my $sync_name (sort keys%{$cfg->{$source}}) {
-           next if $cfg->{$source}->{$sync_name}->{status} ne 'ok';
-           $text .= "$PROG_PATH sync";
-           $text .= " -source  ";
-           if ($cfg->{$source}->{$sync_name}->{vmid}) {
-               $text .= "$cfg->{$source}->{$sync_name}->{source_ip}:" if $cfg->{$source}->{$sync_name}->{source_ip};
-               $text .= "$cfg->{$source}->{$sync_name}->{vmid} ";
-           } else {
-               $text .= "$cfg->{$source}->{$sync_name}->{source_ip}:" if $cfg->{$source}->{$sync_name}->{source_ip};
-               $text .= "$cfg->{$source}->{$sync_name}->{source_pool}";
-               $text .= "$cfg->{$source}->{$sync_name}->{source_path}" if $cfg->{$source}->{$sync_name}->{source_path};
-           }
-           $text .= " -dest  ";
-           $text .= "$cfg->{$source}->{$sync_name}->{dest_ip}:" if $cfg->{$source}->{$sync_name}->{dest_ip};
-           $text .= "$cfg->{$source}->{$sync_name}->{dest_pool}";
-           $text .= "$cfg->{$source}->{$sync_name}->{dest_path}" if $cfg->{$source}->{$sync_name}->{dest_path};
-           $text .= " -name $sync_name ";
-           $text .= " -limit $cfg->{$source}->{$sync_name}->{limit}" if $cfg->{$source}->{$sync_name}->{limit};
-           $text .= " -maxsnap $cfg->{$source}->{$sync_name}->{maxsnap}" if $cfg->{$source}->{$sync_name}->{maxsnap};
-           $text .= "\n";
-       }
-    }
-    die "Can't write to cron\n" if (!print($fh $text));
-    close($fh);
-}
-

 sub get_disks {
     my ($target, $user) = @_;
 
 sub get_disks {
     my ($target, $user) = @_;
 


@@ -830,7 +885,7 @@ sub parse_disks {
        my $line = $1;
 
        next if $line =~ /media=cdrom/;
        my $line = $1;
 
        next if $line =~ /media=cdrom/;

-       next if $line !~ m/^(?:((?:virtio|ide|scsi|sata|mp)\d+)|rootfs): /;
+       next if $line !~ m/$DISK_KEY_RE/;

 
        #QEMU if backup is not set include in  sync
        next if $vm_type eq 'qemu' && ($line =~ m/backup=(?i:0|no|off|false)/);
 
        #QEMU if backup is not set include in  sync
        next if $vm_type eq 'qemu' && ($line =~ m/backup=(?i:0|no|off|false)/);


@@ -840,11 +895,11 @@ sub parse_disks {
 
        my $disk = undef;
        my $stor = undef;
 
        my $disk = undef;
        my $stor = undef;

-       if($line =~ m/^(?:(?:(?:virtio|ide|scsi|sata|mp)\d+)|rootfs): (.*)$/) {
+       if($line =~ m/$DISK_KEY_RE(.*)$/) {

            my @parameter = split(/,/,$1);
 
            foreach my $opt (@parameter) {
            my @parameter = split(/,/,$1);
 
            foreach my $opt (@parameter) {

-               if ($opt =~ m/^(?:file=|volume=)?([^:]+:)([A-Za-z0-9\-]+)$/){
+               if ($opt =~ m/^(?:file=|volume=)?([^:]+):([A-Za-z0-9\-]+)$/){

                    $disk = $2;
                    $stor = $1;
                    last;
                    $disk = $2;
                    $stor = $1;
                    last;


@@ -858,11 +913,13 @@ sub parse_disks {
 
        my $cmd = [];
        push @$cmd, 'ssh', "$user\@$ip", '--' if $ip;
 
        my $cmd = [];
        push @$cmd, 'ssh', "$user\@$ip", '--' if $ip;

-       push @$cmd, 'pvesm', 'path', "$stor$disk";
+       push @$cmd, 'pvesm', 'path', "$stor:$disk";

        my $path = run_cmd($cmd);
 
        my $path = run_cmd($cmd);
 

-       die "Get no path from pvesm path $stor$disk\n" if !$path;
- 
+       die "Get no path from pvesm path $stor:$disk\n" if !$path;
+
+       $disks->{$num}->{storage_id} = $stor;
+

        if ($vm_type eq 'qemu' && $path =~ m/^\/dev\/zvol\/(\w+.*)(\/$disk)$/) {
 
            my @array = split('/', $1);
        if ($vm_type eq 'qemu' && $path =~ m/^\/dev\/zvol\/(\w+.*)(\/$disk)$/) {
 
            my @array = split('/', $1);


@@ -892,23 +949,53 @@ sub parse_disks {
            $num++;
 
        } else {
            $num++;
 
        } else {

-           die "ERROR: in path\n";
+           die "unexpected path '$path'\n";

        }
     }
 
        }
     }
 

-    die "Vm include no disk on zfs.\n" if !$disks->{0};
+    die "Guest does not include any ZFS volumes (or all are excluded by the backup flag).\n"
+       if !$disks->{0};

     return $disks;
 }
 
     return $disks;
 }
 

+# how the corresponding dataset is named on the target
+sub target_dataset {
+    my ($source, $dest) = @_;
+
+    my $target = "$dest->{all}";
+    $target .= "/$dest->{prepend}" if defined($dest->{prepend});
+    $target .= "/$source->{last_part}" if $source->{last_part};
+    $target =~ s!/+!/!g;
+
+    return $target;
+}
+
+# create the parent dataset for the actual target
+sub prepare_prepended_target {
+    my ($source, $dest, $dest_user) = @_;
+
+    die "internal error - not a prepended target\n" if !defined($dest->{prepend});
+
+    # The parent dataset shouldn't be the actual target.
+    die "internal error - no last_part for source\n" if !$source->{last_part};
+
+    my $target = "$dest->{all}/$dest->{prepend}";
+    $target =~ s!/+!/!g;
+
+    return if check_dataset_exists($target, $dest->{ip}, $dest_user);
+
+    create_file_system($target, $dest->{ip}, $dest_user);
+}
+

 sub snapshot_destroy {
 sub snapshot_destroy {

-    my ($source, $dest, $method, $snap, $source_user, $dest_user) = @_;
+    my ($dataset, $snap, $ip, $user) = @_;

 
     my @zfscmd = ('zfs', 'destroy');
 
     my @zfscmd = ('zfs', 'destroy');

-    my $snapshot = "$source->{all}\@$snap";
+    my $snapshot = "$dataset\@$snap";

 
     eval {
 
     eval {

-       if($source->{ip} && $method eq 'ssh'){
-           run_cmd(['ssh', "$source_user\@$source->{ip}", '--', @zfscmd, $snapshot]);
+       if ($ip) {
+           run_cmd(['ssh', "$user\@$ip", '--', @zfscmd, $snapshot]);

        } else {
            run_cmd([@zfscmd, $snapshot]);
        }
        } else {
            run_cmd([@zfscmd, $snapshot]);
        }


@@ -916,46 +1003,27 @@ sub snapshot_destroy {
     if (my $erro = $@) {
        warn "WARN: $erro";
     }
     if (my $erro = $@) {
        warn "WARN: $erro";
     }

-    if ($dest) {
-       my @ssh = $dest->{ip} ? ('ssh', "$dest_user\@$dest->{ip}", '--') : ();
-
-       my $path = "$dest->{all}";
-       $path .= "/$source->{last_part}" if $source->{last_part};
-
-       eval {
-           run_cmd([@ssh, @zfscmd, "$path\@$snap"]);
-       };
-       if (my $erro = $@) {
-           warn "WARN: $erro";
-       }
-    }

 }
 
 }
 

+# check if snapshot for incremental sync exist on source side

 sub snapshot_exist {
 sub snapshot_exist {

-    my ($source , $dest, $method, $dest_user) = @_;
+    my ($source , $dest, $method, $source_user) = @_;

 
     my $cmd = [];
 
     my $cmd = [];

-    push @$cmd, 'ssh', "$dest_user\@$dest->{ip}", '--' if $dest->{ip};
+    push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--' if $source->{ip};

     push @$cmd, 'zfs', 'list', '-rt', 'snapshot', '-Ho', 'name';
 
     push @$cmd, 'zfs', 'list', '-rt', 'snapshot', '-Ho', 'name';
 

-    my $path = $dest->{all};
-    $path .= "/$source->{last_part}" if $source->{last_part};
-    $path .= "\@$source->{old_snap}";
+    my $path = $source->{all};
+    $path .= "\@$dest->{last_snap}";

 
     push @$cmd, $path;
 
 
     push @$cmd, $path;
 

-
-    my $text = "";
-    eval {$text =run_cmd($cmd);};
+    eval {run_cmd($cmd)};

     if (my $erro =$@) {
        warn "WARN: $erro";
        return undef;
     }
     if (my $erro =$@) {
        warn "WARN: $erro";
        return undef;
     }

-
-    while ($text && $text =~ s/^(.*?)(\n|$)//) {
-       my $line =$1;
-       return 1 if $line =~ m/^.*$source->{old_snap}$/;
-    }
+    return 1;

 }
 
 sub send_image {
 }
 
 sub send_image {


@@ -965,11 +1033,13 @@ sub send_image {
 
     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{source_user}\@$source->{ip}", '--' if $source->{ip};
     push @$cmd, 'zfs', 'send';
 
     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{source_user}\@$source->{ip}", '--' if $source->{ip};
     push @$cmd, 'zfs', 'send';

+    push @$cmd, '-L', if $param->{compressed}; # no effect if dataset never had large recordsize
+    push @$cmd, '-c', if $param->{compressed};

     push @$cmd, '-p', if $param->{properties};
     push @$cmd, '-v' if $param->{verbose};
 
     push @$cmd, '-p', if $param->{properties};
     push @$cmd, '-v' if $param->{verbose};
 

-    if($source->{last_snap} && snapshot_exist($source , $dest, $param->{method}, $param->{dest_user})) {
-       push @$cmd, '-i', "$source->{all}\@$source->{last_snap}";
+    if($dest->{last_snap} && snapshot_exist($source , $dest, $param->{method}, $param->{source_user})) {
+       push @$cmd, '-i', "$source->{all}\@$dest->{last_snap}";

     }
     push @$cmd, '--', "$source->{all}\@$source->{new_snap}";
 
     }
     push @$cmd, '--', "$source->{all}\@$source->{new_snap}";
 


@@ -977,9 +1047,7 @@ sub send_image {
        my $bwl = $param->{limit}*1024;
        push @$cmd, \'|', 'cstream', '-t', $bwl;
     }
        my $bwl = $param->{limit}*1024;
        push @$cmd, \'|', 'cstream', '-t', $bwl;
     }

-    my $target = "$dest->{all}";
-    $target .= "/$source->{last_part}" if $source->{last_part};
-    $target =~ s!/+!/!g;
+    my $target = target_dataset($source, $dest);

 
     push @$cmd, \'|';
     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{dest_user}\@$dest->{ip}", '--' if $dest->{ip};
 
     push @$cmd, \'|';
     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{dest_user}\@$dest->{ip}", '--' if $dest->{ip};


@@ -991,7 +1059,7 @@ sub send_image {
     };
 
     if (my $erro = $@) {
     };
 
     if (my $erro = $@) {

-       snapshot_destroy($source, undef, $param->{method}, $source->{new_snap}, $param->{source_user}, $param->{dest_user});
+       snapshot_destroy($source->{all}, $source->{new_snap}, $source->{ip}, $param->{source_user});

        die $erro;
     };
 }
        die $erro;
     };
 }


@@ -1020,8 +1088,8 @@ sub send_config{
            run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", $dest_target_new]);
        }
 
            run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", $dest_target_new]);
        }
 

-       if ($source->{destroy}){
-           my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.$source->{old_snap}";
+       for my $old_snap (@{$dest->{old_snap}}) {
+           my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.${old_snap}";

            if($dest->{ip}){
                run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'rm', '-f', '--', $dest_target_old]);
            } else {
            if($dest->{ip}){
                run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'rm', '-f', '--', $dest_target_old]);
            } else {


@@ -1083,108 +1151,112 @@ sub disable_job {
 
 my $cmd_help = {
     destroy => qq{
 
 my $cmd_help = {
     destroy => qq{

-$PROGNAME destroy -source <string> [OPTIONS]
-
-        remove a sync Job from the scheduler
+$PROGNAME destroy --source <string> [OPTIONS]

 
 

-        -name      string
-
-               name of the sync job, if not set it is default
+    Remove a sync Job from the scheduler

 
 

-        -source    string
+       --name      string
+               The name of the sync job, if not set 'default' is used.

 
 

-                the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
+       --source    string
+               The source can be an  <VMID> or [IP:]<ZFSPool>[/Path]

     },
     create => qq{
     },
     create => qq{

-$PROGNAME create -dest <string> -source <string> [OPTIONS]
+$PROGNAME create --dest <string> --source <string> [OPTIONS]

 
 

-        Create a sync Job
+    Create a new sync-job

 
 

-        -dest      string
+       --dest      string
+               The destination target is like [IP]:<Pool>[/Path]

 
 

-               the destination target is like [IP]:<Pool>[/Path]
+       --dest-user string
+               The name of the user on the destination target, root by default

 
 

-        -dest-user string
-
-               name of the user on the destination target, root by default
-
-        -limit     integer
-
-               max sync speed in kBytes/s, default unlimited
-
-        -maxsnap   string
-
-               how much snapshots will be kept before get erased, default 1
-
-        -name      string
-
-               name of the sync job, if not set it is default
+       --limit     integer
+               Maximal sync speed in kBytes/s, default is unlimited

 
 

-        -skip      boolean
+       --maxsnap   integer
+               The number of snapshots to keep until older ones are erased.
+               The default is 1, use 0 for unlimited.

 
 

-               if this flag is set it will skip the first sync
+       --dest-maxsnap   integer
+               Override maxsnap for the destination dataset.

 
 

-        -source    string
+       --name      string
+               The name of the sync job, if not set it is default

 
 

-               the source can be an <VMID> or [IP:]<ZFSPool>[/Path]
+       --prepend-storage-id
+               If specified, prepend the storage ID to the destination's path(s).

 
 

-        -source-user    string
+       --skip
+               If specified, skip the first sync.

 
 

-               name of the user on the source target, root by default
+       --source    string
+               The source can be an <VMID> or [IP:]<ZFSPool>[/Path]

 
 

-       -properties     boolean
+       --source-user    string
+               The (ssh) user-name on the source target, root by default

 
 

-               Include the dataset's properties in the stream.
+       --compressed
+               If specified, send data without decompressing first. If features lz4_compress,
+               zstd_compress or large_blocks are in use by the source, they need to be enabled on
+               the target as well.

 
 

-       -dest-config-path    string
+       --properties
+               If specified, include the dataset's properties in the stream.

 
 

-               specify a custom config path on the destination target. default is /var/lib/pve-zsync
+       --dest-config-path    string
+               Specifies a custom config path on the destination target.
+               The default is /var/lib/pve-zsync

     },
     sync => qq{
     },
     sync => qq{

-$PROGNAME sync -dest <string> -source <string> [OPTIONS]\n
+$PROGNAME sync --dest <string> --source <string> [OPTIONS]\n

 
 

-       will sync one time
+    Trigger one sync.

 
 

-        -dest      string
+       --dest      string
+               The destination target is like [IP:]<Pool>[/Path]

 
 

-               the destination target is like [IP:]<Pool>[/Path]
+       --dest-user string
+               The (ssh) user-name on the destination target, root by default

 
 

-        -dest-user string
+       --limit     integer
+               The maximal sync speed in kBytes/s, default is unlimited

 
 

-               name of the user on the destination target, root by default
+       --maxsnap   integer
+               The number of snapshots to keep until older ones are erased.
+               The default is 1, use 0 for unlimited.

 
 

-       -limit     integer
+       --dest-maxsnap   integer
+               Override maxsnap for the destination dataset.

 
 

-               max sync speed in kBytes/s, default unlimited
-
-        -maxsnap   integer
-
-               how much snapshots will be kept before get erased, default 1
-
-        -name      string
-
-               name of the sync job, if not set it is default.
+       --name      string
+               The name of the sync job, if not set it is 'default'.

                It is only necessary if scheduler allready contains this source.
 
                It is only necessary if scheduler allready contains this source.
 

-        -source    string
-
-               the source can be an <VMID> or [IP:]<ZFSPool>[/Path]
+       --prepend-storage-id
+               If specified, prepend the storage ID to the destination's path(s).

 
 

-        -source-user    string
+       --source    string
+               The source can either be an <VMID> or [IP:]<ZFSPool>[/Path]

 
 

-               name of the user on the source target, root by default
+       --source-user    string
+               The name of the user on the source target, root by default

 
 

-       -verbose   boolean
+       --verbose
+               If specified, print out the sync progress.

 
 

-               print out the sync progress.
+       --compressed
+               If specified, send data without decompressing first. If features lz4_compress,
+               zstd_compress or large_blocks are in use by the source, they need to be enabled on
+               the target as well.

 
 

-       -properties     boolean
+       --properties
+               If specified, include the dataset's properties in the stream.

 
 

-               Include the dataset's properties in the stream.
-
-       -dest-config-path    string
-
-               specify a custom config path on the destination target. default is /var/lib/pve-zsync
+       --dest-config-path    string
+               Specifies a custom config path on the destination target.
+               The default is /var/lib/pve-zsync

     },
     list => qq{
 $PROGNAME list
     },
     list => qq{
 $PROGNAME list


@@ -1199,43 +1271,37 @@ $PROGNAME status
     help => qq{
 $PROGNAME help <cmd> [OPTIONS]
 
     help => qq{
 $PROGNAME help <cmd> [OPTIONS]
 

-       Get help about specified command.
-
-        <cmd>      string
-
-               Command name
+    Get help about specified command.

 
 

-       -verbose   boolean
+       <cmd>      string
+               Command name to get help about.

 
 

+       --verbose

                Verbose output format.
     },
     enable => qq{
                Verbose output format.
     },
     enable => qq{

-$PROGNAME enable -source <string> [OPTIONS]
+$PROGNAME enable --source <string> [OPTIONS]

 
 

-        enable a syncjob and reset error
-
-        -name      string
+    Enable a sync-job and reset all job-errors, if any.

 
 

+       --name      string

                name of the sync job, if not set it is default
 
                name of the sync job, if not set it is default
 

-        -source    string
-
-                the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
+        --source    string
+               the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]

     },
     disable => qq{
     },
     disable => qq{

-$PROGNAME disable -source <string> [OPTIONS]
-
-        pause a sync job
-
-        -name      string
+$PROGNAME disable --source <string> [OPTIONS]

 
 

-               name of the sync job, if not set it is default
+    Disables (pauses) a sync-job

 
 

-        -source    string
+       --name      string
+               name of the sync-job, if not set it is default

 
 

-                the source can be an  <VMID> or [IP:]<ZFSPool>[/Path] 
+       --source    string
+               the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]

     },
     },

-    printpod => 'internal command',
+    printpod => "$PROGNAME printpod\n\n\tinternal command",

 
 };
 
 
 };
 


@@ -1318,13 +1384,13 @@ sub usage {
     print("ERROR:\tno command specified\n") if !$help;
     print("USAGE:\t$PROGNAME <COMMAND> [ARGS] [OPTIONS]\n");
     print("\t$PROGNAME help [<cmd>] [OPTIONS]\n\n");
     print("ERROR:\tno command specified\n") if !$help;
     print("USAGE:\t$PROGNAME <COMMAND> [ARGS] [OPTIONS]\n");
     print("\t$PROGNAME help [<cmd>] [OPTIONS]\n\n");

-    print("\t$PROGNAME create -dest <string> -source <string> [OPTIONS]\n");
-    print("\t$PROGNAME destroy -source <string> [OPTIONS]\n");
-    print("\t$PROGNAME disable -source <string> [OPTIONS]\n");
-    print("\t$PROGNAME enable -source <string> [OPTIONS]\n");
+    print("\t$PROGNAME create --dest <string> --source <string> [OPTIONS]\n");
+    print("\t$PROGNAME destroy --source <string> [OPTIONS]\n");
+    print("\t$PROGNAME disable --source <string> [OPTIONS]\n");
+    print("\t$PROGNAME enable --source <string> [OPTIONS]\n");

     print("\t$PROGNAME list\n");
     print("\t$PROGNAME status\n");
     print("\t$PROGNAME list\n");
     print("\t$PROGNAME status\n");

-    print("\t$PROGNAME sync -dest <string> -source <string> [OPTIONS]\n");
+    print("\t$PROGNAME sync --dest <string> --source <string> [OPTIONS]\n");

 }
 
 sub check_target {
 }
 
 sub check_target {


@@ -1335,57 +1401,62 @@ sub check_target {
 sub print_pod {
 
     my $synopsis = join("\n", sort values %$cmd_help);
 sub print_pod {
 
     my $synopsis = join("\n", sort values %$cmd_help);

+    my $commands = join(", ", sort keys %$cmd_help);

 
     print <<EOF;
 =head1 NAME
 
 
     print <<EOF;
 =head1 NAME
 

-pve-zsync - PVE ZFS Replication Manager
+pve-zsync - PVE ZFS Storage Sync Tool

 
 =head1 SYNOPSIS
 
 pve-zsync <COMMAND> [ARGS] [OPTIONS]
 
 
 =head1 SYNOPSIS
 
 pve-zsync <COMMAND> [ARGS] [OPTIONS]
 

-$synopsis
+Where <COMMAND> can be one of: $commands

 
 =head1 DESCRIPTION
 
 
 =head1 DESCRIPTION
 

-This Tool helps you to sync your VM or directory which stored on ZFS between 2 servers.
-This tool also has the capability to add jobs to cron so the sync will be automatically done.
-The default syncing interval is set to 15 min, if you want to change this value you can do this in /etc/cron.d/pve-zsync.
-To config cron see man crontab.
+The pve-zsync tool can help you to sync your VMs or directories stored on ZFS
+between multiple servers.

 
 

-=head2 PVE ZFS Storage sync Tool
+pve-zsync is able to automatically configure CRON jobs, so that a periodic sync
+will be automatically triggered.
+The default sync interval is 15 min, if you want to change this value you can
+do this in F</etc/cron.d/pve-zsync>. If you need help to configure CRON tabs, see
+man crontab.

 
 

-This Tool can get remote pool on other PVE or send Pool to others ZFS machines
+=head1 COMMANDS AND OPTIONS
+
+$synopsis

 
 =head1 EXAMPLES
 
 
 =head1 EXAMPLES
 

-add sync job from local VM to remote ZFS Server
-pve-zsync create -source=100 -dest=192.168.1.2:zfspool
+Adds a job for syncing the local VM 100 to a remote server's ZFS pool named "tank":
+    pve-zsync create --source=100 -dest=192.168.1.2:tank

 
 =head1 IMPORTANT FILES
 
 
 =head1 IMPORTANT FILES
 

-Cron jobs and config are stored at                      /etc/cron.d/pve-zsync
+Cron jobs and config are stored in F</etc/cron.d/pve-zsync>

 
 

-The VM config get copied on the destination machine to  /var/lib/pve-zsync/
+The VM configuration itself gets copied to the destination machines
+F</var/lib/pve-zsync/> path.

 
 =head1 COPYRIGHT AND DISCLAIMER
 
 
 =head1 COPYRIGHT AND DISCLAIMER
 

-Copyright (C) 2007-2015 Proxmox Server Solutions GmbH
+Copyright (C) 2007-2021 Proxmox Server Solutions GmbH

 
 

-This program is free software: you can redistribute it and/or modify it
-under the terms of the GNU Affero General Public License as published
-by the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU Affero General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.

 
 

-This program is distributed in the hope that it will be useful, but
-WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-Affero General Public License for more details.
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+details.

 
 

-You should have received a copy of the GNU Affero General Public
-License along with this program.  If not, see
-<http://www.gnu.org/licenses/>.
+You should have received a copy of the GNU Affero General Public License along
+with this program. If not, see <http://www.gnu.org/licenses/>.

 
 EOF
 }
 
 EOF
 }