X-Git-Url: https://git.proxmox.com/?p=pve-zsync.git;a=blobdiff_plain;f=pve-zsync;h=425ffa2a5220ec001956e9567b59e9fbfb9a9b01;hp=2c10a4da92804abb2a6df9d7baea63379e572c69;hb=HEAD;hpb=53e7ea54e04a4596ce6373b962da565a2682015a diff --git a/pve-zsync b/pve-zsync index 2c10a4d..de5d46f 100755 --- a/pve-zsync +++ b/pve-zsync @@ -9,6 +9,7 @@ use File::Path qw(make_path); use JSON; use IO::File; use String::ShellQuote 'shell_quote'; +use Text::ParseWords; my $PROGNAME = "pve-zsync"; my $CONFIG_PATH = "/var/lib/${PROGNAME}"; @@ -53,7 +54,7 @@ my $HOSTRE = "(?:$HOSTv4RE1|\\[$IPV6RE\\])"; # ipv6 must always be in brac # targets are either a VMID, or a 'host:zpool/path' with 'host:' being optional my $TARGETRE = qr!^(?:($HOSTRE):)?(\d+|(?:[\w\-_]+)(/.+)?)$!; -my $DISK_KEY_RE = qr/^(?:(?:(?:virtio|ide|scsi|sata|efidisk|mp)\d+)|rootfs): /; +my $DISK_KEY_RE = qr/^(?:(?:(?:virtio|ide|scsi|sata|efidisk|tpmstate|mp)\d+)|rootfs): /; my $INSTANCE_ID = get_instance_id($$); @@ -152,15 +153,15 @@ sub get_status { return undef; } -sub check_pool_exists { - my ($target, $user) = @_; +sub check_dataset_exists { + my ($dataset, $ip, $user) = @_; my $cmd = []; - if ($target->{ip}) { - push @$cmd, 'ssh', "$user\@$target->{ip}", '--'; + if ($ip) { + push @$cmd, 'ssh', "$user\@$ip", '--'; } - push @$cmd, 'zfs', 'list', '-H', '--', $target->{all}; + push @$cmd, 'zfs', 'list', '-H', '--', $dataset; eval { run_cmd($cmd); }; @@ -171,6 +172,19 @@ sub check_pool_exists { return 1; } +sub create_file_system { + my ($file_system, $ip, $user) = @_; + + my $cmd = []; + + if ($ip) { + push @$cmd, 'ssh', "$user\@$ip", '--'; + } + push @$cmd, 'zfs', 'create', $file_system; + + run_cmd($cmd); +} + sub parse_target { my ($text) = @_; @@ -219,7 +233,7 @@ sub read_cron { my $text = read_file($CRONJOBS, 0); - return encode_cron(@{$text}); + return parse_cron(@{$text}); } sub parse_argv { @@ -231,11 +245,14 @@ sub parse_argv { verbose => undef, limit => undef, maxsnap => undef, + dest_maxsnap => undef, name => undef, skip => undef, method => undef, source_user => undef, dest_user => undef, + prepend_storage_id => undef, + compressed => undef, properties => undef, dest_config_path => undef, }; @@ -247,11 +264,14 @@ sub parse_argv { 'verbose' => \$param->{verbose}, 'limit=i' => \$param->{limit}, 'maxsnap=i' => \$param->{maxsnap}, + 'dest-maxsnap=i' => \$param->{dest_maxsnap}, 'name=s' => \$param->{name}, 'skip' => \$param->{skip}, 'method=s' => \$param->{method}, 'source-user=s' => \$param->{source_user}, 'dest-user=s' => \$param->{dest_user}, + 'prepend-storage-id' => \$param->{prepend_storage_id}, + 'compressed' => \$param->{compressed}, 'properties' => \$param->{properties}, 'dest-config-path=s' => \$param->{dest_config_path}, ); @@ -285,14 +305,13 @@ sub add_state_to_job { return $job; } -sub encode_cron { +sub parse_cron { my (@text) = @_; my $cfg = {}; while (my $line = shift(@text)) { - - my @arg = split('\s', $line); + my @arg = Text::ParseWords::shellwords($line); my $param = parse_argv(@arg); if ($param->{source} && $param->{dest}) { @@ -312,17 +331,21 @@ sub param_to_job { my $job = {}; my $source = parse_target($param->{source}); - my $dest = parse_target($param->{dest}) if $param->{dest}; + my $dest; + $dest = parse_target($param->{dest}) if $param->{dest}; $job->{name} = !$param->{name} ? "default" : $param->{name}; $job->{dest} = $param->{dest} if $param->{dest}; $job->{method} = "local" if !$dest->{ip} && !$source->{ip}; $job->{method} = "ssh" if !$job->{method}; $job->{limit} = $param->{limit}; - $job->{maxsnap} = $param->{maxsnap} if $param->{maxsnap}; + $job->{maxsnap} = $param->{maxsnap}; + $job->{dest_maxsnap} = $param->{dest_maxsnap}; $job->{source} = $param->{source}; $job->{source_user} = $param->{source_user}; $job->{dest_user} = $param->{dest_user}; + $job->{prepend_storage_id} = !!$param->{prepend_storage_id}; + $job->{compressed} = !!$param->{compressed}; $job->{properties} = !!$param->{properties}; $job->{dest_config_path} = $param->{dest_config_path} if $param->{dest_config_path}; @@ -421,10 +444,10 @@ sub update_cron { my $new_fh = IO::File->new("> ${CRONJOBS}.new"); die "Could not open file ${CRONJOBS}.new: $!\n" if !$new_fh; - die "can't write to $CRONJOBS.new\n" if !print($new_fh $text); + print $new_fh $text or die "can't write to $CRONJOBS.new: $!\n"; close ($new_fh); - die "can't move $CRONJOBS.new: $!\n" if !rename "${CRONJOBS}.new", $CRONJOBS; + rename "${CRONJOBS}.new", $CRONJOBS or die "can't move $CRONJOBS.new: $!\n"; } sub format_job { @@ -443,11 +466,14 @@ sub format_job { $text .= " root"; $text .= " $PROGNAME sync --source $job->{source} --dest $job->{dest}"; $text .= " --name $job->{name} --maxsnap $job->{maxsnap}"; + $text .= " --dest-maxsnap $job->{dest_maxsnap}" if defined($job->{dest_maxsnap}); $text .= " --limit $job->{limit}" if $job->{limit}; $text .= " --method $job->{method}"; $text .= " --verbose" if $job->{verbose}; $text .= " --source-user $job->{source_user}"; $text .= " --dest-user $job->{dest_user}"; + $text .= " --prepend-storage-id" if $job->{prepend_storage_id}; + $text .= " --compressed" if $job->{compressed}; $text .= " --properties" if $job->{properties}; $text .= " --dest-config-path $job->{dest_config_path}" if $job->{dest_config_path}; $text .= "\n"; @@ -517,10 +543,12 @@ sub init { run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{source_user}\@$ip"]); } - die "Pool $dest->{all} does not exists\n" if !check_pool_exists($dest, $param->{dest_user}); + die "Pool $dest->{all} does not exist\n" + if !check_dataset_exists($dest->{all}, $dest->{ip}, $param->{dest_user}); if (!defined($source->{vmid})) { - die "Pool $source->{all} does not exists\n" if !check_pool_exists($source, $param->{source_user}); + die "Pool $source->{all} does not exist\n" + if !check_dataset_exists($source->{all}, $source->{ip}, $param->{source_user}); } my $vm_type = vm_exists($source, $param->{source_user}); @@ -657,14 +685,37 @@ sub sync { my $sync_path = sub { my ($source, $dest, $job, $param, $date) = @_; - ($dest->{old_snap}, $dest->{last_snap}) = snapshot_get($source, $dest, $param->{maxsnap}, $param->{name}, $param->{dest_user}); + my $dest_dataset = target_dataset($source, $dest); + + ($dest->{old_snap}, $dest->{last_snap}) = snapshot_get( + $dest_dataset, + $param->{dest_maxsnap} // $param->{maxsnap}, + $param->{name}, + $dest->{ip}, + $param->{dest_user}, + ); + + ($source->{old_snap}) = snapshot_get( + $source->{all}, + $param->{maxsnap}, + $param->{name}, + $source->{ip}, + $param->{source_user}, + ); + + prepare_prepended_target($source, $dest, $param->{dest_user}) if defined($dest->{prepend}); snapshot_add($source, $dest, $param->{name}, $date, $param->{source_user}, $param->{dest_user}); send_image($source, $dest, $param); - snapshot_destroy($source, $dest, $param->{method}, $dest->{old_snap}, $param->{source_user}, $param->{dest_user}) if ($source->{destroy} && $dest->{old_snap}); + for my $old_snap (@{$source->{old_snap}}) { + snapshot_destroy($source->{all}, $old_snap, $source->{ip}, $param->{source_user}); + } + for my $old_snap (@{$dest->{old_snap}}) { + snapshot_destroy($dest_dataset, $old_snap, $dest->{ip}, $param->{dest_user}); + } }; eval{ @@ -678,6 +729,10 @@ sub sync { $source->{pool} = $disks->{$disk}->{pool}; $source->{path} = $disks->{$disk}->{path} if $disks->{$disk}->{path}; $source->{last_part} = $disks->{$disk}->{last_part}; + + $dest->{prepend} = $disks->{$disk}->{storage_id} + if $param->{prepend_storage_id}; + &$sync_path($source, $dest, $job, $param, $date); } if ($param->{method} eq "ssh" && ($source->{ip} || $dest->{ip})) { @@ -719,15 +774,12 @@ sub sync { } sub snapshot_get{ - my ($source, $dest, $max_snap, $name, $dest_user) = @_; + my ($dataset, $max_snap, $name, $ip, $user) = @_; my $cmd = []; - push @$cmd, 'ssh', "$dest_user\@$dest->{ip}", '--', if $dest->{ip}; + push @$cmd, 'ssh', "$user\@$ip", '--', if $ip; push @$cmd, 'zfs', 'list', '-r', '-t', 'snapshot', '-Ho', 'name', '-S', 'creation'; - - my $path = $dest->{all}; - $path .= "/$source->{last_part}" if $source->{last_part}; - push @$cmd, $path; + push @$cmd, $dataset; my $raw; eval {$raw = run_cmd($cmd)}; @@ -738,7 +790,7 @@ sub snapshot_get{ my $index = 0; my $line = ""; my $last_snap = undef; - my $old_snap; + my $old_snap = []; while ($raw && $raw =~ s/^(.*?)(\n|$)//) { $line = $1; @@ -746,12 +798,15 @@ sub snapshot_get{ $last_snap = $1 if (!$last_snap); } if ($line =~ m/(rep_\Q${name}\E_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})$/) { - $old_snap = $1; + # interpreted as infinity + last if $max_snap <= 0; + + my $snap = $1; $index++; - if ($index == $max_snap) { - $source->{destroy} = 1; - last; - }; + + if ($index >= $max_snap) { + push @{$old_snap}, $snap; + } } } @@ -777,7 +832,7 @@ sub snapshot_add { }; if (my $err = $@) { - snapshot_destroy($source, $dest, 'ssh', $snap_name, $source_user, $dest_user); + snapshot_destroy($source->{all}, $snap_name, $source->{ip}, $source_user); die "$err\n"; } } @@ -844,7 +899,7 @@ sub parse_disks { my @parameter = split(/,/,$1); foreach my $opt (@parameter) { - if ($opt =~ m/^(?:file=|volume=)?([^:]+:)([A-Za-z0-9\-]+)$/){ + if ($opt =~ m/^(?:file=|volume=)?([^:]+):([A-Za-z0-9\-]+)$/){ $disk = $2; $stor = $1; last; @@ -858,10 +913,12 @@ sub parse_disks { my $cmd = []; push @$cmd, 'ssh', "$user\@$ip", '--' if $ip; - push @$cmd, 'pvesm', 'path', "$stor$disk"; + push @$cmd, 'pvesm', 'path', "$stor:$disk"; my $path = run_cmd($cmd); - die "Get no path from pvesm path $stor$disk\n" if !$path; + die "Get no path from pvesm path $stor:$disk\n" if !$path; + + $disks->{$num}->{storage_id} = $stor; if ($vm_type eq 'qemu' && $path =~ m/^\/dev\/zvol\/(\w+.*)(\/$disk)$/) { @@ -892,23 +949,53 @@ sub parse_disks { $num++; } else { - die "ERROR: in path\n"; + die "unexpected path '$path'\n"; } } - die "Vm include no disk on zfs.\n" if !$disks->{0}; + die "Guest does not include any ZFS volumes (or all are excluded by the backup flag).\n" + if !$disks->{0}; return $disks; } +# how the corresponding dataset is named on the target +sub target_dataset { + my ($source, $dest) = @_; + + my $target = "$dest->{all}"; + $target .= "/$dest->{prepend}" if defined($dest->{prepend}); + $target .= "/$source->{last_part}" if $source->{last_part}; + $target =~ s!/+!/!g; + + return $target; +} + +# create the parent dataset for the actual target +sub prepare_prepended_target { + my ($source, $dest, $dest_user) = @_; + + die "internal error - not a prepended target\n" if !defined($dest->{prepend}); + + # The parent dataset shouldn't be the actual target. + die "internal error - no last_part for source\n" if !$source->{last_part}; + + my $target = "$dest->{all}/$dest->{prepend}"; + $target =~ s!/+!/!g; + + return if check_dataset_exists($target, $dest->{ip}, $dest_user); + + create_file_system($target, $dest->{ip}, $dest_user); +} + sub snapshot_destroy { - my ($source, $dest, $method, $snap, $source_user, $dest_user) = @_; + my ($dataset, $snap, $ip, $user) = @_; my @zfscmd = ('zfs', 'destroy'); - my $snapshot = "$source->{all}\@$snap"; + my $snapshot = "$dataset\@$snap"; eval { - if($source->{ip} && $method eq 'ssh'){ - run_cmd(['ssh', "$source_user\@$source->{ip}", '--', @zfscmd, $snapshot]); + if ($ip) { + run_cmd(['ssh', "$user\@$ip", '--', @zfscmd, $snapshot]); } else { run_cmd([@zfscmd, $snapshot]); } @@ -916,19 +1003,6 @@ sub snapshot_destroy { if (my $erro = $@) { warn "WARN: $erro"; } - if ($dest) { - my @ssh = $dest->{ip} ? ('ssh', "$dest_user\@$dest->{ip}", '--') : (); - - my $path = "$dest->{all}"; - $path .= "/$source->{last_part}" if $source->{last_part}; - - eval { - run_cmd([@ssh, @zfscmd, "$path\@$snap"]); - }; - if (my $erro = $@) { - warn "WARN: $erro"; - } - } } # check if snapshot for incremental sync exist on source side @@ -959,6 +1033,8 @@ sub send_image { push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{source_user}\@$source->{ip}", '--' if $source->{ip}; push @$cmd, 'zfs', 'send'; + push @$cmd, '-L', if $param->{compressed}; # no effect if dataset never had large recordsize + push @$cmd, '-c', if $param->{compressed}; push @$cmd, '-p', if $param->{properties}; push @$cmd, '-v' if $param->{verbose}; @@ -971,9 +1047,7 @@ sub send_image { my $bwl = $param->{limit}*1024; push @$cmd, \'|', 'cstream', '-t', $bwl; } - my $target = "$dest->{all}"; - $target .= "/$source->{last_part}" if $source->{last_part}; - $target =~ s!/+!/!g; + my $target = target_dataset($source, $dest); push @$cmd, \'|'; push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{dest_user}\@$dest->{ip}", '--' if $dest->{ip}; @@ -985,7 +1059,7 @@ sub send_image { }; if (my $erro = $@) { - snapshot_destroy($source, undef, $param->{method}, $source->{new_snap}, $param->{source_user}, $param->{dest_user}); + snapshot_destroy($source->{all}, $source->{new_snap}, $source->{ip}, $param->{source_user}); die $erro; }; } @@ -1014,8 +1088,8 @@ sub send_config{ run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", $dest_target_new]); } - if ($source->{destroy}){ - my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.$dest->{old_snap}"; + for my $old_snap (@{$dest->{old_snap}}) { + my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.${old_snap}"; if($dest->{ip}){ run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'rm', '-f', '--', $dest_target_old]); } else { @@ -1077,108 +1151,112 @@ sub disable_job { my $cmd_help = { destroy => qq{ -$PROGNAME destroy -source [OPTIONS] +$PROGNAME destroy --source [OPTIONS] - remove a sync Job from the scheduler + Remove a sync Job from the scheduler - -name string + --name string + The name of the sync job, if not set 'default' is used. - name of the sync job, if not set it is default - - -source string - - the source can be an or [IP:][/Path] + --source string + The source can be an or [IP:][/Path] }, create => qq{ -$PROGNAME create -dest -source [OPTIONS] - - Create a sync Job +$PROGNAME create --dest --source [OPTIONS] - -dest string + Create a new sync-job - the destination target is like [IP]:[/Path] + --dest string + The destination target is like [IP]:[/Path] - -dest-user string + --dest-user string + The name of the user on the destination target, root by default - name of the user on the destination target, root by default + --limit integer + Maximal sync speed in kBytes/s, default is unlimited - -limit integer + --maxsnap integer + The number of snapshots to keep until older ones are erased. + The default is 1, use 0 for unlimited. - max sync speed in kBytes/s, default unlimited + --dest-maxsnap integer + Override maxsnap for the destination dataset. - -maxsnap string + --name string + The name of the sync job, if not set it is default - how much snapshots will be kept before get erased, default 1 - - -name string - - name of the sync job, if not set it is default - - -skip + --prepend-storage-id + If specified, prepend the storage ID to the destination's path(s). + --skip If specified, skip the first sync. - -source string - - the source can be an or [IP:][/Path] - - -source-user string + --source string + The source can be an or [IP:][/Path] - name of the user on the source target, root by default + --source-user string + The (ssh) user-name on the source target, root by default - -properties + --compressed + If specified, send data without decompressing first. If features lz4_compress, + zstd_compress or large_blocks are in use by the source, they need to be enabled on + the target as well. + --properties If specified, include the dataset's properties in the stream. - -dest-config-path string - - specify a custom config path on the destination target. default is /var/lib/pve-zsync + --dest-config-path string + Specifies a custom config path on the destination target. + The default is /var/lib/pve-zsync }, sync => qq{ -$PROGNAME sync -dest -source [OPTIONS]\n - - will sync one time - - -dest string - - the destination target is like [IP:][/Path] - - -dest-user string +$PROGNAME sync --dest --source [OPTIONS]\n - name of the user on the destination target, root by default + Trigger one sync. - -limit integer + --dest string + The destination target is like [IP:][/Path] - max sync speed in kBytes/s, default unlimited + --dest-user string + The (ssh) user-name on the destination target, root by default - -maxsnap integer + --limit integer + The maximal sync speed in kBytes/s, default is unlimited - how much snapshots will be kept before get erased, default 1 + --maxsnap integer + The number of snapshots to keep until older ones are erased. + The default is 1, use 0 for unlimited. - -name string + --dest-maxsnap integer + Override maxsnap for the destination dataset. - name of the sync job, if not set it is default. + --name string + The name of the sync job, if not set it is 'default'. It is only necessary if scheduler allready contains this source. - -source string + --prepend-storage-id + If specified, prepend the storage ID to the destination's path(s). - the source can be an or [IP:][/Path] + --source string + The source can either be an or [IP:][/Path] - -source-user string - - name of the user on the source target, root by default - - -verbose + --source-user string + The name of the user on the source target, root by default + --verbose If specified, print out the sync progress. - -properties + --compressed + If specified, send data without decompressing first. If features lz4_compress, + zstd_compress or large_blocks are in use by the source, they need to be enabled on + the target as well. + --properties If specified, include the dataset's properties in the stream. - -dest-config-path string - - specify a custom config path on the destination target. default is /var/lib/pve-zsync + --dest-config-path string + Specifies a custom config path on the destination target. + The default is /var/lib/pve-zsync }, list => qq{ $PROGNAME list @@ -1193,43 +1271,37 @@ $PROGNAME status help => qq{ $PROGNAME help [OPTIONS] - Get help about specified command. - - string + Get help about specified command. - Command name - - -verbose + string + Command name to get help about. + --verbose Verbose output format. }, enable => qq{ -$PROGNAME enable -source [OPTIONS] - - enable a syncjob and reset error +$PROGNAME enable --source [OPTIONS] - -name string + Enable a sync-job and reset all job-errors, if any. + --name string name of the sync job, if not set it is default - -source string - - the source can be an or [IP:][/Path] + --source string + the source can be an or [IP:][/Path] }, disable => qq{ -$PROGNAME disable -source [OPTIONS] +$PROGNAME disable --source [OPTIONS] - pause a sync job + Disables (pauses) a sync-job - -name string - - name of the sync job, if not set it is default + --name string + name of the sync-job, if not set it is default - -source string - - the source can be an or [IP:][/Path] + --source string + the source can be an or [IP:][/Path] }, - printpod => 'internal command', + printpod => "$PROGNAME printpod\n\n\tinternal command", }; @@ -1312,13 +1384,13 @@ sub usage { print("ERROR:\tno command specified\n") if !$help; print("USAGE:\t$PROGNAME [ARGS] [OPTIONS]\n"); print("\t$PROGNAME help [] [OPTIONS]\n\n"); - print("\t$PROGNAME create -dest -source [OPTIONS]\n"); - print("\t$PROGNAME destroy -source [OPTIONS]\n"); - print("\t$PROGNAME disable -source [OPTIONS]\n"); - print("\t$PROGNAME enable -source [OPTIONS]\n"); + print("\t$PROGNAME create --dest --source [OPTIONS]\n"); + print("\t$PROGNAME destroy --source [OPTIONS]\n"); + print("\t$PROGNAME disable --source [OPTIONS]\n"); + print("\t$PROGNAME enable --source [OPTIONS]\n"); print("\t$PROGNAME list\n"); print("\t$PROGNAME status\n"); - print("\t$PROGNAME sync -dest -source [OPTIONS]\n"); + print("\t$PROGNAME sync --dest --source [OPTIONS]\n"); } sub check_target { @@ -1329,57 +1401,62 @@ sub check_target { sub print_pod { my $synopsis = join("\n", sort values %$cmd_help); + my $commands = join(", ", sort keys %$cmd_help); print < [ARGS] [OPTIONS] -$synopsis +Where can be one of: $commands =head1 DESCRIPTION -This Tool helps you to sync your VM or directory which stored on ZFS between 2 servers. -This tool also has the capability to add jobs to cron so the sync will be automatically done. -The default syncing interval is set to 15 min, if you want to change this value you can do this in /etc/cron.d/pve-zsync. -To config cron see man crontab. +The pve-zsync tool can help you to sync your VMs or directories stored on ZFS +between multiple servers. -=head2 PVE ZFS Storage sync Tool +pve-zsync is able to automatically configure CRON jobs, so that a periodic sync +will be automatically triggered. +The default sync interval is 15 min, if you want to change this value you can +do this in F. If you need help to configure CRON tabs, see +man crontab. -This Tool can get remote pool on other PVE or send Pool to others ZFS machines +=head1 COMMANDS AND OPTIONS + +$synopsis =head1 EXAMPLES -add sync job from local VM to remote ZFS Server -pve-zsync create -source=100 -dest=192.168.1.2:zfspool +Adds a job for syncing the local VM 100 to a remote server's ZFS pool named "tank": + pve-zsync create --source=100 -dest=192.168.1.2:tank =head1 IMPORTANT FILES -Cron jobs and config are stored at /etc/cron.d/pve-zsync +Cron jobs and config are stored in F -The VM config get copied on the destination machine to /var/lib/pve-zsync/ +The VM configuration itself gets copied to the destination machines +F path. =head1 COPYRIGHT AND DISCLAIMER Copyright (C) 2007-2021 Proxmox Server Solutions GmbH -This program is free software: you can redistribute it and/or modify it -under the terms of the GNU Affero General Public License as published -by the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU Affero General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) any +later version. -This program is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Affero General Public License for more details. +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU Affero General Public License for more +details. -You should have received a copy of the GNU Affero General Public -License along with this program. If not, see -. +You should have received a copy of the GNU Affero General Public License along +with this program. If not, see . EOF }