pve-zsync

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl qw(:flock SEEK_END);
   7 use Getopt::Long qw(GetOptionsFromArray);
   8 use File::Path qw(make_path);
   9 use JSON;
  10 use IO::File;
  11 use String::ShellQuote 'shell_quote';
  12
  13 my $PROGNAME = "pve-zsync";
  14 my $CONFIG_PATH = "/var/lib/${PROGNAME}";
  15 my $STATE = "${CONFIG_PATH}/sync_state";
  16 my $CRONJOBS = "/etc/cron.d/$PROGNAME";
  17 my $PATH = "/usr/sbin";
  18 my $PVE_DIR = "/etc/pve/local";
  19 my $QEMU_CONF = "${PVE_DIR}/qemu-server";
  20 my $LXC_CONF = "${PVE_DIR}/lxc";
  21 my $PROG_PATH = "$PATH/${PROGNAME}";
  22 my $INTERVAL = 15;
  23 my $DEBUG;
  24
  25 BEGIN {
  26     $DEBUG = 0; # change default here. not above on declaration!
  27     $DEBUG ||= $ENV{ZSYNC_DEBUG};
  28     if ($DEBUG) {
  29         require Data::Dumper;
  30         Data::Dumper->import();
  31     }
  32 }
  33
  34 my $IPV4OCTET = "(?:25[0-5]|(?:[1-9]|1[0-9]|2[0-4])?[0-9])";
  35 my $IPV4RE = "(?:(?:$IPV4OCTET\\.){3}$IPV4OCTET)";
  36 my $IPV6H16 = "(?:[0-9a-fA-F]{1,4})";
  37 my $IPV6LS32 = "(?:(?:$IPV4RE|$IPV6H16:$IPV6H16))";
  38
  39 my $IPV6RE = "(?:" .
  40     "(?:(?:" .                             "(?:$IPV6H16:){6})$IPV6LS32)|" .
  41     "(?:(?:" .                           "::(?:$IPV6H16:){5})$IPV6LS32)|" .
  42     "(?:(?:(?:" .              "$IPV6H16)?::(?:$IPV6H16:){4})$IPV6LS32)|" .
  43     "(?:(?:(?:(?:$IPV6H16:){0,1}$IPV6H16)?::(?:$IPV6H16:){3})$IPV6LS32)|" .
  44     "(?:(?:(?:(?:$IPV6H16:){0,2}$IPV6H16)?::(?:$IPV6H16:){2})$IPV6LS32)|" .
  45     "(?:(?:(?:(?:$IPV6H16:){0,3}$IPV6H16)?::(?:$IPV6H16:){1})$IPV6LS32)|" .
  46     "(?:(?:(?:(?:$IPV6H16:){0,4}$IPV6H16)?::" .           ")$IPV6LS32)|" .
  47     "(?:(?:(?:(?:$IPV6H16:){0,5}$IPV6H16)?::" .            ")$IPV6H16)|" .
  48     "(?:(?:(?:(?:$IPV6H16:){0,6}$IPV6H16)?::" .                    ")))";
  49
  50 my $HOSTv4RE0 = "(?:[\\w\\.\\-_]+|$IPV4RE)";       # hostname or ipv4 address
  51 my $HOSTv4RE1 = "(?:$HOSTv4RE0|\\[$HOSTv4RE0\\])"; # these may be in brackets, too
  52 my $HOSTRE = "(?:$HOSTv4RE1|\\[$IPV6RE\\])";       # ipv6 must always be in brackets
  53 # targets are either a VMID, or a 'host:zpool/path' with 'host:' being optional
  54 my $TARGETRE = qr!^(?:($HOSTRE):)?(\d+|(?:[\w\-_]+)(/.+)?)$!;
  55
  56 my $DISK_KEY_RE = qr/^(?:(?:(?:virtio|ide|scsi|sata|efidisk|mp)\d+)|rootfs): /;
  57
  58 my $INSTANCE_ID = get_instance_id($$);
  59
  60 my $command = $ARGV[0];
  61
  62 if (defined($command) && $command ne 'help' && $command ne 'printpod') {
  63     check_bin ('cstream');
  64     check_bin ('zfs');
  65     check_bin ('ssh');
  66     check_bin ('scp');
  67 }
  68
  69 $SIG{TERM} = $SIG{QUIT} = $SIG{PIPE} = $SIG{HUP} = $SIG{KILL} = $SIG{INT} =  sub {
  70     die "Signaled, aborting sync: $!\n";
  71 };
  72
  73 sub check_bin {
  74     my ($bin)  = @_;
  75
  76     foreach my $p (split (/:/, $ENV{PATH})) {
  77         my $fn = "$p/$bin";
  78         if (-x $fn) {
  79             return $fn;
  80         }
  81     }
  82
  83     die "unable to find command '$bin'\n";
  84 }
  85
  86 sub read_file {
  87     my ($filename, $one_line_only) = @_;
  88
  89     my $fh = IO::File->new($filename, "r")
  90         or die "Could not open file ${filename}: $!\n";
  91
  92     my $text = $one_line_only ? <$fh> : [ <$fh> ];
  93
  94     close($fh);
  95
  96     return $text;
  97 }
  98
  99 sub cut_target_width {
 100     my ($path, $maxlen) = @_;
 101     $path =~ s@/+@/@g;
 102
 103     return $path if length($path) <= $maxlen;
 104
 105     return '..'.substr($path, -$maxlen+2) if $path !~ m@/@;
 106
 107     $path =~ s@/([^/]+/?)$@@;
 108     my $tail = $1;
 109
 110     if (length($tail)+3 == $maxlen) {
 111         return "../$tail";
 112     } elsif (length($tail)+2 >= $maxlen) {
 113         return '..'.substr($tail, -$maxlen+2)
 114     }
 115
 116     $path =~ s@(/[^/]+)(?:/|$)@@;
 117     my $head = $1;
 118     my $both = length($head) + length($tail);
 119     my $remaining = $maxlen-$both-4; # -4 for "/../"
 120
 121     if ($remaining < 0) {
 122         return substr($head, 0, $maxlen - length($tail) - 3) . "../$tail"; # -3 for "../"
 123     }
 124
 125     substr($path, ($remaining/2), (length($path)-$remaining), '..');
 126     return "$head/" . $path . "/$tail";
 127 }
 128
 129 sub locked {
 130     my ($lock_fn, $code) = @_;
 131
 132     my $lock_fh = IO::File->new("> $lock_fn");
 133
 134     flock($lock_fh, LOCK_EX) || die "Couldn't acquire lock - $!\n";
 135     my $res = eval { $code->() };
 136     my $err = $@;
 137
 138     flock($lock_fh, LOCK_UN) || warn "Error unlocking - $!\n";
 139     die "$err" if $err;
 140
 141     close($lock_fh);
 142     return $res;
 143 }
 144
 145 sub get_status {
 146     my ($source, $name, $status) = @_;
 147
 148     if ($status->{$source->{all}}->{$name}->{status}) {
 149         return $status;
 150     }
 151
 152     return undef;
 153 }
 154
 155 sub check_dataset_exists {
 156     my ($dataset, $ip, $user) = @_;
 157
 158     my $cmd = [];
 159
 160     if ($ip) {
 161         push @$cmd, 'ssh', "$user\@$ip", '--';
 162     }
 163     push @$cmd, 'zfs', 'list', '-H', '--', $dataset;
 164     eval {
 165         run_cmd($cmd);
 166     };
 167
 168     if ($@) {
 169         return 0;
 170     }
 171     return 1;
 172 }
 173
 174 sub create_file_system {
 175     my ($file_system, $ip, $user) = @_;
 176
 177     my $cmd = [];
 178
 179     if ($ip) {
 180         push @$cmd, 'ssh', "$user\@$ip", '--';
 181     }
 182     push @$cmd, 'zfs', 'create', $file_system;
 183
 184     run_cmd($cmd);
 185 }
 186
 187 sub parse_target {
 188     my ($text) = @_;
 189
 190     my $errstr = "$text : is not a valid input! Use [IP:]<VMID> or [IP:]<ZFSPool>[/Path]";
 191     my $target = {};
 192
 193     if ($text !~ $TARGETRE) {
 194         die "$errstr\n";
 195     }
 196     $target->{all} = $2;
 197     $target->{ip} = $1 if $1;
 198     my @parts = split('/', $2);
 199
 200     $target->{ip} =~ s/^\[(.*)\]$/$1/ if $target->{ip};
 201
 202     my $pool = $target->{pool} = shift(@parts);
 203     die "$errstr\n" if !$pool;
 204
 205     if ($pool =~ m/^\d+$/) {
 206         $target->{vmid} = $pool;
 207         delete $target->{pool};
 208     }
 209
 210     return $target if (@parts == 0);
 211     $target->{last_part} = pop(@parts);
 212
 213     if ($target->{ip}) {
 214         pop(@parts);
 215     }
 216     if (@parts > 0) {
 217         $target->{path} = join('/', @parts);
 218     }
 219
 220     return $target;
 221 }
 222
 223 sub read_cron {
 224
 225     #This is for the first use to init file;
 226     if (!-e $CRONJOBS) {
 227         my $new_fh = IO::File->new("> $CRONJOBS");
 228         die "Could not create $CRONJOBS: $!\n" if !$new_fh;
 229         close($new_fh);
 230         return undef;
 231     }
 232
 233     my $text = read_file($CRONJOBS, 0);
 234
 235     return encode_cron(@{$text});
 236 }
 237
 238 sub parse_argv {
 239     my (@arg) = @_;
 240
 241     my $param = {
 242         dest => undef,
 243         source => undef,
 244         verbose => undef,
 245         limit => undef,
 246         maxsnap => undef,
 247         name => undef,
 248         skip => undef,
 249         method => undef,
 250         source_user => undef,
 251         dest_user => undef,
 252         prepend_storage_id => undef,
 253         properties => undef,
 254         dest_config_path => undef,
 255     };
 256
 257     my ($ret) = GetOptionsFromArray(
 258         \@arg,
 259         'dest=s' => \$param->{dest},
 260         'source=s' => \$param->{source},
 261         'verbose' => \$param->{verbose},
 262         'limit=i' => \$param->{limit},
 263         'maxsnap=i' => \$param->{maxsnap},
 264         'name=s' => \$param->{name},
 265         'skip' => \$param->{skip},
 266         'method=s' => \$param->{method},
 267         'source-user=s' => \$param->{source_user},
 268         'dest-user=s' => \$param->{dest_user},
 269         'prepend-storage-id' => \$param->{prepend_storage_id},
 270         'properties' => \$param->{properties},
 271         'dest-config-path=s' => \$param->{dest_config_path},
 272     );
 273
 274     die "can't parse options\n" if $ret == 0;
 275
 276     $param->{name} //= "default";
 277     $param->{maxsnap} //= 1;
 278     $param->{method} //= "ssh";
 279     $param->{source_user} //= "root";
 280     $param->{dest_user} //= "root";
 281
 282     return $param;
 283 }
 284
 285 sub add_state_to_job {
 286     my ($job) = @_;
 287
 288     my $states = read_state();
 289     my $state = $states->{$job->{source}}->{$job->{name}};
 290
 291     $job->{state} = $state->{state};
 292     $job->{lsync} = $state->{lsync};
 293     $job->{vm_type} = $state->{vm_type};
 294     $job->{instance_id} = $state->{instance_id};
 295
 296     for (my $i = 0; $state->{"snap$i"}; $i++) {
 297         $job->{"snap$i"} = $state->{"snap$i"};
 298     }
 299
 300     return $job;
 301 }
 302
 303 sub encode_cron {
 304     my (@text) = @_;
 305
 306     my $cfg = {};
 307
 308     while (my $line = shift(@text)) {
 309
 310         my @arg = split('\s', $line);
 311         my $param = parse_argv(@arg);
 312
 313         if ($param->{source} && $param->{dest}) {
 314             my $source = delete $param->{source};
 315             my $name = delete $param->{name};
 316
 317             $cfg->{$source}->{$name} = $param;
 318         }
 319     }
 320
 321     return $cfg;
 322 }
 323
 324 sub param_to_job {
 325     my ($param) = @_;
 326
 327     my $job = {};
 328
 329     my $source = parse_target($param->{source});
 330     my $dest;
 331     $dest = parse_target($param->{dest}) if $param->{dest};
 332
 333     $job->{name} = !$param->{name} ? "default" : $param->{name};
 334     $job->{dest} = $param->{dest} if $param->{dest};
 335     $job->{method} = "local" if !$dest->{ip} && !$source->{ip};
 336     $job->{method} = "ssh" if !$job->{method};
 337     $job->{limit} = $param->{limit};
 338     $job->{maxsnap} = $param->{maxsnap};
 339     $job->{source} = $param->{source};
 340     $job->{source_user} = $param->{source_user};
 341     $job->{dest_user} = $param->{dest_user};
 342     $job->{prepend_storage_id} = !!$param->{prepend_storage_id};
 343     $job->{properties} = !!$param->{properties};
 344     $job->{dest_config_path} = $param->{dest_config_path} if $param->{dest_config_path};
 345
 346     return $job;
 347 }
 348
 349 sub read_state {
 350
 351     if (!-e $STATE) {
 352         make_path $CONFIG_PATH;
 353         my $new_fh = IO::File->new("> $STATE");
 354         die "Could not create $STATE: $!\n" if !$new_fh;
 355         print $new_fh "{}";
 356         close($new_fh);
 357         return undef;
 358     }
 359
 360     my $text = read_file($STATE, 1);
 361     return decode_json($text);
 362 }
 363
 364 sub update_state {
 365     my ($job) = @_;
 366
 367     my $text = eval { read_file($STATE, 1); };
 368
 369     my $out_fh = IO::File->new("> $STATE.new");
 370     die "Could not open file ${STATE}.new: $!\n" if !$out_fh;
 371
 372     my $states = {};
 373     my $state = {};
 374     if ($text){
 375         $states = decode_json($text);
 376         $state = $states->{$job->{source}}->{$job->{name}};
 377     }
 378
 379     if ($job->{state} ne "del") {
 380         $state->{state} = $job->{state};
 381         $state->{lsync} = $job->{lsync};
 382         $state->{instance_id} = $job->{instance_id};
 383         $state->{vm_type} = $job->{vm_type};
 384
 385         for (my $i = 0; $job->{"snap$i"} ; $i++) {
 386             $state->{"snap$i"} = $job->{"snap$i"};
 387         }
 388         $states->{$job->{source}}->{$job->{name}} = $state;
 389     } else {
 390
 391         delete $states->{$job->{source}}->{$job->{name}};
 392         delete $states->{$job->{source}} if !keys %{$states->{$job->{source}}};
 393     }
 394
 395     $text = encode_json($states);
 396     print $out_fh $text;
 397
 398     close($out_fh);
 399     rename "$STATE.new", $STATE;
 400
 401     return $states;
 402 }
 403
 404 sub update_cron {
 405     my ($job) = @_;
 406
 407     my $updated;
 408     my $has_header;
 409     my $line_no = 0;
 410     my $text = "";
 411     my $header = "SHELL=/bin/sh\n";
 412     $header .= "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n\n";
 413
 414     my $current = read_file($CRONJOBS, 0);
 415
 416     foreach my $line (@{$current}) {
 417         chomp($line);
 418         if ($line =~ m/source $job->{source} .*name $job->{name} /) {
 419             $updated = 1;
 420             next if $job->{state} eq "del";
 421             $text .= format_job($job, $line);
 422         } else {
 423             if (($line_no < 3) && ($line =~ /^(PATH|SHELL)/ )) {
 424                 $has_header = 1;
 425             }
 426             $text .= "$line\n";
 427         }
 428         $line_no++;
 429     }
 430
 431     if (!$has_header) {
 432         $text = "$header$text";
 433     }
 434
 435     if (!$updated) {
 436         $text .= format_job($job);
 437     }
 438     my $new_fh = IO::File->new("> ${CRONJOBS}.new");
 439     die "Could not open file ${CRONJOBS}.new: $!\n" if !$new_fh;
 440
 441     print $new_fh $text or die "can't write to $CRONJOBS.new: $!\n";
 442     close ($new_fh);
 443
 444     rename "${CRONJOBS}.new", $CRONJOBS or die "can't move $CRONJOBS.new: $!\n";
 445 }
 446
 447 sub format_job {
 448     my ($job, $line) = @_;
 449     my $text = "";
 450
 451     if ($job->{state} eq "stopped") {
 452         $text = "#";
 453     }
 454     if ($line) {
 455         $line =~ /^#*\s*((?:\S+\s+){4}\S+)\s+root/;
 456         $text .= $1;
 457     } else {
 458         $text .= "*/$INTERVAL * * * *";
 459     }
 460     $text .= " root";
 461     $text .= " $PROGNAME sync --source $job->{source} --dest $job->{dest}";
 462     $text .= " --name $job->{name} --maxsnap $job->{maxsnap}";
 463     $text .= " --limit $job->{limit}" if $job->{limit};
 464     $text .= " --method $job->{method}";
 465     $text .= " --verbose" if $job->{verbose};
 466     $text .= " --source-user $job->{source_user}";
 467     $text .= " --dest-user $job->{dest_user}";
 468     $text .= " --prepend-storage-id" if $job->{prepend_storage_id};
 469     $text .= " --properties" if $job->{properties};
 470     $text .= " --dest-config-path $job->{dest_config_path}" if $job->{dest_config_path};
 471     $text .= "\n";
 472
 473     return $text;
 474 }
 475
 476 sub list {
 477
 478     my $cfg = read_cron();
 479
 480     my $list = sprintf("%-25s%-25s%-10s%-20s%-6s%-5s\n" , "SOURCE", "NAME", "STATE", "LAST SYNC", "TYPE", "CON");
 481
 482     my $states = read_state();
 483     foreach my $source (sort keys%{$cfg}) {
 484         foreach my $name (sort keys%{$cfg->{$source}}) {
 485             $list .= sprintf("%-25s", cut_target_width($source, 25));
 486             $list .= sprintf("%-25s", cut_target_width($name, 25));
 487             $list .= sprintf("%-10s", $states->{$source}->{$name}->{state});
 488             $list .= sprintf("%-20s", $states->{$source}->{$name}->{lsync});
 489             $list .= sprintf("%-6s", defined($states->{$source}->{$name}->{vm_type}) ? $states->{$source}->{$name}->{vm_type} : "undef");
 490             $list .= sprintf("%-5s\n", $cfg->{$source}->{$name}->{method});
 491         }
 492     }
 493
 494     return $list;
 495 }
 496
 497 sub vm_exists {
 498     my ($target, $user) = @_;
 499
 500     return undef if !defined($target->{vmid});
 501
 502     my $conf_fn = "$target->{vmid}.conf";
 503
 504     if ($target->{ip}) {
 505         my @cmd = ('ssh', "$user\@$target->{ip}", '--', '/bin/ls');
 506         return "qemu" if eval { run_cmd([@cmd, "$QEMU_CONF/$conf_fn"]) };
 507         return "lxc" if  eval { run_cmd([@cmd, "$LXC_CONF/$conf_fn"]) };
 508     } else {
 509         return "qemu" if -f "$QEMU_CONF/$conf_fn";
 510         return "lxc" if -f "$LXC_CONF/$conf_fn";
 511     }
 512
 513     return undef;
 514 }
 515
 516 sub init {
 517     my ($param) = @_;
 518
 519     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 520         my $cfg = read_cron();
 521
 522         my $job = param_to_job($param);
 523
 524         $job->{state} = "ok";
 525         $job->{lsync} = 0;
 526
 527         my $source = parse_target($param->{source});
 528         my $dest = parse_target($param->{dest});
 529
 530         if (my $ip =  $dest->{ip}) {
 531             run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{dest_user}\@$ip"]);
 532         }
 533
 534         if (my $ip =  $source->{ip}) {
 535             run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{source_user}\@$ip"]);
 536         }
 537
 538         die "Pool $dest->{all} does not exist\n"
 539             if !check_dataset_exists($dest->{all}, $dest->{ip}, $param->{dest_user});
 540
 541         if (!defined($source->{vmid})) {
 542             die "Pool $source->{all} does not exist\n"
 543                 if !check_dataset_exists($source->{all}, $source->{ip}, $param->{source_user});
 544         }
 545
 546         my $vm_type = vm_exists($source, $param->{source_user});
 547         $job->{vm_type} = $vm_type;
 548         $source->{vm_type} = $vm_type;
 549
 550         die "VM $source->{vmid} doesn't exist\n" if $source->{vmid} && !$vm_type;
 551
 552         die "Config already exists\n" if $cfg->{$job->{source}}->{$job->{name}};
 553
 554         #check if vm has zfs disks if not die;
 555         get_disks($source, $param->{source_user}) if $source->{vmid};
 556
 557         update_cron($job);
 558         update_state($job);
 559     }); #cron and state lock
 560
 561     return if $param->{skip};
 562
 563     eval { sync($param) };
 564     if (my $err = $@) {
 565         destroy_job($param);
 566         print $err;
 567     }
 568 }
 569
 570 sub get_job {
 571     my ($param) = @_;
 572
 573     my $cfg = read_cron();
 574
 575     if (!$cfg->{$param->{source}}->{$param->{name}}) {
 576         die "Job  with source $param->{source} and name $param->{name} does not exist\n" ;
 577     }
 578     my $job = $cfg->{$param->{source}}->{$param->{name}};
 579     $job->{name} = $param->{name};
 580     $job->{source} = $param->{source};
 581     $job = add_state_to_job($job);
 582
 583     return $job;
 584 }
 585
 586 sub destroy_job {
 587     my ($param) = @_;
 588
 589     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 590         my $job = get_job($param);
 591         $job->{state} = "del";
 592
 593         update_cron($job);
 594         update_state($job);
 595     });
 596 }
 597
 598 sub get_instance_id {
 599     my ($pid) = @_;
 600
 601     my $stat = read_file("/proc/$pid/stat", 1)
 602         or die "unable to read process stats\n";
 603     my $boot_id = read_file("/proc/sys/kernel/random/boot_id", 1)
 604         or die "unable to read boot ID\n";
 605
 606     my $stats = [ split(/\s+/, $stat) ];
 607     my $starttime = $stats->[21];
 608     chomp($boot_id);
 609
 610     return "${pid}:${starttime}:${boot_id}";
 611 }
 612
 613 sub instance_exists {
 614     my ($instance_id) = @_;
 615
 616     if (defined($instance_id) && $instance_id =~ m/^([1-9][0-9]*):/) {
 617         my $pid = $1;
 618         my $actual_id = eval { get_instance_id($pid); };
 619         return defined($actual_id) && $actual_id eq $instance_id;
 620     }
 621
 622     return 0;
 623 }
 624
 625 sub sync {
 626     my ($param) = @_;
 627
 628     my $job;
 629
 630     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 631         eval { $job = get_job($param) };
 632
 633         if ($job) {
 634             my $state = $job->{state} // 'ok';
 635             $state = 'ok' if !instance_exists($job->{instance_id});
 636
 637             if ($state eq "syncing" || $state eq "waiting") {
 638                 die "Job --source $param->{source} --name $param->{name} is already scheduled to sync\n";
 639             }
 640
 641             $job->{state} = "waiting";
 642             $job->{instance_id} = $INSTANCE_ID;
 643
 644             update_state($job);
 645         }
 646     });
 647
 648     locked("$CONFIG_PATH/sync.lock", sub {
 649
 650         my $date = get_date();
 651
 652         my $dest;
 653         my $source;
 654         my $vm_type;
 655
 656         locked("$CONFIG_PATH/cron_and_state.lock", sub {
 657             #job might've changed while we waited for the sync lock, but we can be sure it's not syncing
 658             eval { $job = get_job($param); };
 659
 660             if ($job && defined($job->{state}) && $job->{state} eq "stopped") {
 661                 die "Job --source $param->{source} --name $param->{name} has been disabled\n";
 662             }
 663
 664             $dest = parse_target($param->{dest});
 665             $source = parse_target($param->{source});
 666
 667             $vm_type = vm_exists($source, $param->{source_user});
 668             $source->{vm_type} = $vm_type;
 669
 670             if ($job) {
 671                 $job->{state} = "syncing";
 672                 $job->{vm_type} = $vm_type if !$job->{vm_type};
 673                 update_state($job);
 674             }
 675         }); #cron and state lock
 676
 677         my $sync_path = sub {
 678             my ($source, $dest, $job, $param, $date) = @_;
 679
 680             ($dest->{old_snap}, $dest->{last_snap}) = snapshot_get($source, $dest, $param->{maxsnap}, $param->{name}, $param->{dest_user});
 681
 682             prepare_prepended_target($source, $dest, $param->{dest_user}) if defined($dest->{prepend});
 683
 684             snapshot_add($source, $dest, $param->{name}, $date, $param->{source_user}, $param->{dest_user});
 685
 686             send_image($source, $dest, $param);
 687
 688             for my $old_snap (@{$dest->{old_snap}}) {
 689                 snapshot_destroy($source, $dest, $param->{method}, $old_snap, $param->{source_user}, $param->{dest_user});
 690             }
 691         };
 692
 693         eval{
 694             if ($source->{vmid}) {
 695                 die "VM $source->{vmid} doesn't exist\n" if !$vm_type;
 696                 die "source-user has to be root for syncing VMs\n" if ($param->{source_user} ne "root");
 697                 my $disks = get_disks($source, $param->{source_user});
 698
 699                 foreach my $disk (sort keys %{$disks}) {
 700                     $source->{all} = $disks->{$disk}->{all};
 701                     $source->{pool} = $disks->{$disk}->{pool};
 702                     $source->{path} = $disks->{$disk}->{path} if $disks->{$disk}->{path};
 703                     $source->{last_part} = $disks->{$disk}->{last_part};
 704
 705                     $dest->{prepend} = $disks->{$disk}->{storage_id}
 706                         if $param->{prepend_storage_id};
 707
 708                     &$sync_path($source, $dest, $job, $param, $date);
 709                 }
 710                 if ($param->{method} eq "ssh" && ($source->{ip} || $dest->{ip})) {
 711                     send_config($source, $dest,'ssh', $param->{source_user}, $param->{dest_user}, $param->{dest_config_path});
 712                 } else {
 713                     send_config($source, $dest,'local', $param->{source_user}, $param->{dest_user}, $param->{dest_config_path});
 714                 }
 715             } else {
 716                 &$sync_path($source, $dest, $job, $param, $date);
 717             }
 718         };
 719         if (my $err = $@) {
 720             locked("$CONFIG_PATH/cron_and_state.lock", sub {
 721                 eval { $job = get_job($param); };
 722                 if ($job) {
 723                     $job->{state} = "error";
 724                     delete $job->{instance_id};
 725                     update_state($job);
 726                 }
 727             });
 728             print "Job --source $param->{source} --name $param->{name} got an ERROR!!!\nERROR Message:\n";
 729             die "$err\n";
 730         }
 731
 732         locked("$CONFIG_PATH/cron_and_state.lock", sub {
 733             eval { $job = get_job($param); };
 734             if ($job) {
 735                 if (defined($job->{state}) && $job->{state} eq "stopped") {
 736                     $job->{state} = "stopped";
 737                 } else {
 738                     $job->{state} = "ok";
 739                 }
 740                 $job->{lsync} = $date;
 741                 delete $job->{instance_id};
 742                 update_state($job);
 743             }
 744         });
 745     }); #sync lock
 746 }
 747
 748 sub snapshot_get{
 749     my ($source, $dest, $max_snap, $name, $dest_user) = @_;
 750
 751     my $cmd = [];
 752     push @$cmd, 'ssh', "$dest_user\@$dest->{ip}", '--', if $dest->{ip};
 753     push @$cmd, 'zfs', 'list', '-r', '-t', 'snapshot', '-Ho', 'name', '-S', 'creation';
 754
 755     my $path = target_dataset($source, $dest);
 756     push @$cmd, $path;
 757
 758     my $raw;
 759     eval {$raw = run_cmd($cmd)};
 760     if (my $erro =$@) { #this means the volume doesn't exist on dest yet
 761        return undef;
 762     }
 763
 764     my $index = 0;
 765     my $line = "";
 766     my $last_snap = undef;
 767     my $old_snap = [];
 768
 769     while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
 770         $line = $1;
 771         if ($line =~ m/@(.*)$/) {
 772             $last_snap = $1 if (!$last_snap);
 773         }
 774         if ($line =~ m/(rep_\Q${name}\E_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})$/) {
 775             # interpreted as infinity
 776             last if $max_snap <= 0;
 777
 778             my $snap = $1;
 779             $index++;
 780
 781             if ($index >= $max_snap) {
 782                 push @{$old_snap}, $snap;
 783             }
 784         }
 785     }
 786
 787     return ($old_snap, $last_snap) if $last_snap;
 788
 789     return undef;
 790 }
 791
 792 sub snapshot_add {
 793     my ($source, $dest, $name, $date, $source_user, $dest_user) = @_;
 794
 795     my $snap_name = "rep_$name\_".$date;
 796
 797     $source->{new_snap} = $snap_name;
 798
 799     my $path = "$source->{all}\@$snap_name";
 800
 801     my $cmd = [];
 802     push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--', if $source->{ip};
 803     push @$cmd, 'zfs', 'snapshot', $path;
 804     eval{
 805         run_cmd($cmd);
 806     };
 807
 808     if (my $err = $@) {
 809         snapshot_destroy($source, $dest, 'ssh', $snap_name, $source_user, $dest_user);
 810         die "$err\n";
 811     }
 812 }
 813
 814 sub get_disks {
 815     my ($target, $user) = @_;
 816
 817     my $cmd = [];
 818     push @$cmd, 'ssh', "$user\@$target->{ip}", '--', if $target->{ip};
 819
 820     if ($target->{vm_type} eq 'qemu') {
 821         push @$cmd, 'qm', 'config', $target->{vmid};
 822     } elsif ($target->{vm_type} eq 'lxc') {
 823         push @$cmd, 'pct', 'config', $target->{vmid};
 824     } else {
 825         die "VM Type unknown\n";
 826     }
 827
 828     my $res = run_cmd($cmd);
 829
 830     my $disks = parse_disks($res, $target->{ip}, $target->{vm_type}, $user);
 831
 832     return $disks;
 833 }
 834
 835 sub run_cmd {
 836     my ($cmd) = @_;
 837     print "Start CMD\n" if $DEBUG;
 838     print Dumper $cmd if $DEBUG;
 839     if (ref($cmd) eq 'ARRAY') {
 840         $cmd = join(' ', map { ref($_) ? $$_ : shell_quote($_) } @$cmd);
 841     }
 842     my $output = `$cmd 2>&1`;
 843
 844     die "COMMAND:\n\t$cmd\nGET ERROR:\n\t$output" if 0 != $?;
 845
 846     chomp($output);
 847     print Dumper $output if $DEBUG;
 848     print "END CMD\n" if $DEBUG;
 849     return $output;
 850 }
 851
 852 sub parse_disks {
 853     my ($text, $ip, $vm_type, $user) = @_;
 854
 855     my $disks;
 856
 857     my $num = 0;
 858     while ($text && $text =~ s/^(.*?)(\n|$)//) {
 859         my $line = $1;
 860
 861         next if $line =~ /media=cdrom/;
 862         next if $line !~ m/$DISK_KEY_RE/;
 863
 864         #QEMU if backup is not set include in  sync
 865         next if $vm_type eq 'qemu' && ($line =~ m/backup=(?i:0|no|off|false)/);
 866
 867         #LXC if backup is not set do no in sync
 868         next if $vm_type eq 'lxc' && ($line =~ m/^mp\d:/) && ($line !~ m/backup=(?i:1|yes|on|true)/);
 869
 870         my $disk = undef;
 871         my $stor = undef;
 872         if($line =~ m/$DISK_KEY_RE(.*)$/) {
 873             my @parameter = split(/,/,$1);
 874
 875             foreach my $opt (@parameter) {
 876                 if ($opt =~ m/^(?:file=|volume=)?([^:]+):([A-Za-z0-9\-]+)$/){
 877                     $disk = $2;
 878                     $stor = $1;
 879                     last;
 880                 }
 881             }
 882         }
 883         if (!defined($disk) || !defined($stor)) {
 884             print "Disk: \"$line\" has no valid zfs dataset format and will be skipped\n";
 885             next;
 886         }
 887
 888         my $cmd = [];
 889         push @$cmd, 'ssh', "$user\@$ip", '--' if $ip;
 890         push @$cmd, 'pvesm', 'path', "$stor:$disk";
 891         my $path = run_cmd($cmd);
 892
 893         die "Get no path from pvesm path $stor:$disk\n" if !$path;
 894
 895         $disks->{$num}->{storage_id} = $stor;
 896
 897         if ($vm_type eq 'qemu' && $path =~ m/^\/dev\/zvol\/(\w+.*)(\/$disk)$/) {
 898
 899             my @array = split('/', $1);
 900             $disks->{$num}->{pool} = shift(@array);
 901             $disks->{$num}->{all} = $disks->{$num}->{pool};
 902             if (0 < @array) {
 903                 $disks->{$num}->{path} = join('/', @array);
 904                 $disks->{$num}->{all} .= "\/$disks->{$num}->{path}";
 905             }
 906             $disks->{$num}->{last_part} = $disk;
 907             $disks->{$num}->{all} .= "\/$disk";
 908
 909             $num++;
 910         } elsif ($vm_type eq 'lxc' && $path =~ m/^\/(\w+.+)(\/(\w+.*))*(\/$disk)$/) {
 911
 912             $disks->{$num}->{pool} = $1;
 913             $disks->{$num}->{all} = $disks->{$num}->{pool};
 914
 915             if ($2) {
 916                 $disks->{$num}->{path} = $3;
 917                 $disks->{$num}->{all} .= "\/$disks->{$num}->{path}";
 918             }
 919
 920             $disks->{$num}->{last_part} = $disk;
 921             $disks->{$num}->{all} .= "\/$disk";
 922
 923             $num++;
 924
 925         } else {
 926             die "ERROR: in path\n";
 927         }
 928     }
 929
 930     die "Vm include no disk on zfs.\n" if !$disks->{0};
 931     return $disks;
 932 }
 933
 934 # how the corresponding dataset is named on the target
 935 sub target_dataset {
 936     my ($source, $dest) = @_;
 937
 938     my $target = "$dest->{all}";
 939     $target .= "/$dest->{prepend}" if defined($dest->{prepend});
 940     $target .= "/$source->{last_part}" if $source->{last_part};
 941     $target =~ s!/+!/!g;
 942
 943     return $target;
 944 }
 945
 946 # create the parent dataset for the actual target
 947 sub prepare_prepended_target {
 948     my ($source, $dest, $dest_user) = @_;
 949
 950     die "internal error - not a prepended target\n" if !defined($dest->{prepend});
 951
 952     # The parent dataset shouldn't be the actual target.
 953     die "internal error - no last_part for source\n" if !$source->{last_part};
 954
 955     my $target = "$dest->{all}/$dest->{prepend}";
 956     $target =~ s!/+!/!g;
 957
 958     return if check_dataset_exists($target, $dest->{ip}, $dest_user);
 959
 960     create_file_system($target, $dest->{ip}, $dest_user);
 961 }
 962
 963 sub snapshot_destroy {
 964     my ($source, $dest, $method, $snap, $source_user, $dest_user) = @_;
 965
 966     my @zfscmd = ('zfs', 'destroy');
 967     my $snapshot = "$source->{all}\@$snap";
 968
 969     eval {
 970         if($source->{ip} && $method eq 'ssh'){
 971             run_cmd(['ssh', "$source_user\@$source->{ip}", '--', @zfscmd, $snapshot]);
 972         } else {
 973             run_cmd([@zfscmd, $snapshot]);
 974         }
 975     };
 976     if (my $erro = $@) {
 977         warn "WARN: $erro";
 978     }
 979     if ($dest) {
 980         my @ssh = $dest->{ip} ? ('ssh', "$dest_user\@$dest->{ip}", '--') : ();
 981
 982         my $path = target_dataset($source, $dest);
 983
 984         eval {
 985             run_cmd([@ssh, @zfscmd, "$path\@$snap"]);
 986         };
 987         if (my $erro = $@) {
 988             warn "WARN: $erro";
 989         }
 990     }
 991 }
 992
 993 # check if snapshot for incremental sync exist on source side
 994 sub snapshot_exist {
 995     my ($source , $dest, $method, $source_user) = @_;
 996
 997     my $cmd = [];
 998     push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--' if $source->{ip};
 999     push @$cmd, 'zfs', 'list', '-rt', 'snapshot', '-Ho', 'name';
1000
1001     my $path = $source->{all};
1002     $path .= "\@$dest->{last_snap}";
1003
1004     push @$cmd, $path;
1005
1006     eval {run_cmd($cmd)};
1007     if (my $erro =$@) {
1008         warn "WARN: $erro";
1009         return undef;
1010     }
1011     return 1;
1012 }
1013
1014 sub send_image {
1015     my ($source, $dest, $param) = @_;
1016
1017     my $cmd = [];
1018
1019     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{source_user}\@$source->{ip}", '--' if $source->{ip};
1020     push @$cmd, 'zfs', 'send';
1021     push @$cmd, '-p', if $param->{properties};
1022     push @$cmd, '-v' if $param->{verbose};
1023
1024     if($dest->{last_snap} && snapshot_exist($source , $dest, $param->{method}, $param->{source_user})) {
1025         push @$cmd, '-i', "$source->{all}\@$dest->{last_snap}";
1026     }
1027     push @$cmd, '--', "$source->{all}\@$source->{new_snap}";
1028
1029     if ($param->{limit}){
1030         my $bwl = $param->{limit}*1024;
1031         push @$cmd, \'|', 'cstream', '-t', $bwl;
1032     }
1033     my $target = target_dataset($source, $dest);
1034
1035     push @$cmd, \'|';
1036     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{dest_user}\@$dest->{ip}", '--' if $dest->{ip};
1037     push @$cmd, 'zfs', 'recv', '-F', '--';
1038     push @$cmd, "$target";
1039
1040     eval {
1041         run_cmd($cmd)
1042     };
1043
1044     if (my $erro = $@) {
1045         snapshot_destroy($source, undef, $param->{method}, $source->{new_snap}, $param->{source_user}, $param->{dest_user});
1046         die $erro;
1047     };
1048 }
1049
1050
1051 sub send_config{
1052     my ($source, $dest, $method, $source_user, $dest_user, $dest_config_path) = @_;
1053
1054     my $source_target = $source->{vm_type} eq 'qemu' ? "$QEMU_CONF/$source->{vmid}.conf": "$LXC_CONF/$source->{vmid}.conf";
1055     my $dest_target_new ="$source->{vmid}.conf.$source->{vm_type}.$source->{new_snap}";
1056
1057     my $config_dir = $dest_config_path // $CONFIG_PATH;
1058     $config_dir .= "/$dest->{last_part}" if $dest->{last_part};
1059
1060     $dest_target_new = $config_dir.'/'.$dest_target_new;
1061
1062     if ($method eq 'ssh'){
1063         if ($dest->{ip} && $source->{ip}) {
1064             run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'mkdir', '-p', '--', $config_dir]);
1065             run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", "$dest_user\@[$dest->{ip}]:$dest_target_new"]);
1066         } elsif ($dest->{ip}) {
1067             run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'mkdir', '-p', '--', $config_dir]);
1068             run_cmd(['scp', '--', $source_target, "$dest_user\@[$dest->{ip}]:$dest_target_new"]);
1069         } elsif ($source->{ip}) {
1070             run_cmd(['mkdir', '-p', '--', $config_dir]);
1071             run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", $dest_target_new]);
1072         }
1073
1074         for my $old_snap (@{$dest->{old_snap}}) {
1075             my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.${old_snap}";
1076             if($dest->{ip}){
1077                 run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'rm', '-f', '--', $dest_target_old]);
1078             } else {
1079                 run_cmd(['rm', '-f', '--', $dest_target_old]);
1080             }
1081         }
1082     } elsif ($method eq 'local') {
1083         run_cmd(['mkdir', '-p', '--', $config_dir]);
1084         run_cmd(['cp', $source_target, $dest_target_new]);
1085     }
1086 }
1087
1088 sub get_date {
1089     my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
1090     my $datestamp = sprintf ("%04d-%02d-%02d_%02d:%02d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
1091
1092     return $datestamp;
1093 }
1094
1095 sub status {
1096     my $cfg = read_cron();
1097
1098     my $status_list = sprintf("%-25s%-25s%-10s\n", "SOURCE", "NAME", "STATUS");
1099
1100     my $states = read_state();
1101
1102     foreach my $source (sort keys%{$cfg}) {
1103         foreach my $sync_name (sort keys%{$cfg->{$source}}) {
1104             $status_list .= sprintf("%-25s", cut_target_width($source, 25));
1105             $status_list .= sprintf("%-25s", cut_target_width($sync_name, 25));
1106             $status_list .= "$states->{$source}->{$sync_name}->{state}\n";
1107         }
1108     }
1109
1110     return $status_list;
1111 }
1112
1113 sub enable_job {
1114     my ($param) = @_;
1115
1116     locked("$CONFIG_PATH/cron_and_state.lock", sub {
1117         my $job = get_job($param);
1118         $job->{state} = "ok";
1119         update_state($job);
1120         update_cron($job);
1121     });
1122 }
1123
1124 sub disable_job {
1125     my ($param) = @_;
1126
1127     locked("$CONFIG_PATH/cron_and_state.lock", sub {
1128         my $job = get_job($param);
1129         $job->{state} = "stopped";
1130         update_state($job);
1131         update_cron($job);
1132     });
1133 }
1134
1135 my $cmd_help = {
1136     destroy => qq{
1137 $PROGNAME destroy --source <string> [OPTIONS]
1138
1139     Remove a sync Job from the scheduler
1140
1141         --name      string
1142                 The name of the sync job, if not set 'default' is used.
1143
1144         --source    string
1145                 The source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1146     },
1147     create => qq{
1148 $PROGNAME create --dest <string> --source <string> [OPTIONS]
1149
1150     Create a new sync-job
1151
1152         --dest      string
1153                 The destination target is like [IP]:<Pool>[/Path]
1154
1155         --dest-user string
1156                 The name of the user on the destination target, root by default
1157
1158         --limit     integer
1159                 Maximal sync speed in kBytes/s, default is unlimited
1160
1161         --maxsnap   integer
1162                 The number of snapshots to keep until older ones are erased.
1163                 The default is 1, use 0 for unlimited.
1164
1165         --name      string
1166                 The name of the sync job, if not set it is default
1167
1168         --prepend-storage-id
1169                 If specified, prepend the storage ID to the destination's path(s).
1170
1171         --skip
1172                 If specified, skip the first sync.
1173
1174         --source    string
1175                 The source can be an <VMID> or [IP:]<ZFSPool>[/Path]
1176
1177         --source-user    string
1178                 The (ssh) user-name on the source target, root by default
1179
1180         --properties
1181                 If specified, include the dataset's properties in the stream.
1182
1183         --dest-config-path    string
1184                 Specifies a custom config path on the destination target.
1185                 The default is /var/lib/pve-zsync
1186     },
1187     sync => qq{
1188 $PROGNAME sync --dest <string> --source <string> [OPTIONS]\n
1189
1190     Trigger one sync.
1191
1192         --dest      string
1193                 The destination target is like [IP:]<Pool>[/Path]
1194
1195         --dest-user string
1196                 The (ssh) user-name on the destination target, root by default
1197
1198         --limit     integer
1199                 The maximal sync speed in kBytes/s, default is unlimited
1200
1201         --maxsnap   integer
1202                 The number of snapshots to keep until older ones are erased.
1203                 The default is 1, use 0 for unlimited.
1204
1205         --name      string
1206                 The name of the sync job, if not set it is 'default'.
1207                 It is only necessary if scheduler allready contains this source.
1208
1209         --prepend-storage-id
1210                 If specified, prepend the storage ID to the destination's path(s).
1211
1212         --source    string
1213                 The source can either be an <VMID> or [IP:]<ZFSPool>[/Path]
1214
1215         --source-user    string
1216                 The name of the user on the source target, root by default
1217
1218         --verbose
1219                 If specified, print out the sync progress.
1220
1221         --properties
1222                 If specified, include the dataset's properties in the stream.
1223
1224         --dest-config-path    string
1225                 Specifies a custom config path on the destination target.
1226                 The default is /var/lib/pve-zsync
1227     },
1228     list => qq{
1229 $PROGNAME list
1230
1231         Get a List of all scheduled Sync Jobs
1232     },
1233     status => qq{
1234 $PROGNAME status
1235
1236         Get the status of all scheduled Sync Jobs
1237     },
1238     help => qq{
1239 $PROGNAME help <cmd> [OPTIONS]
1240
1241     Get help about specified command.
1242
1243         <cmd>      string
1244                 Command name to get help about.
1245
1246         --verbose
1247                 Verbose output format.
1248     },
1249     enable => qq{
1250 $PROGNAME enable --source <string> [OPTIONS]
1251
1252     Enable a sync-job and reset all job-errors, if any.
1253
1254         --name      string
1255                 name of the sync job, if not set it is default
1256
1257         --source    string
1258                 the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1259     },
1260     disable => qq{
1261 $PROGNAME disable --source <string> [OPTIONS]
1262
1263     Disables (pauses) a sync-job
1264
1265         --name      string
1266                 name of the sync-job, if not set it is default
1267
1268         --source    string
1269                 the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1270     },
1271     printpod => "$PROGNAME printpod\n\n\tinternal command",
1272
1273 };
1274
1275 if (!$command) {
1276     usage(); die "\n";
1277 } elsif (!$cmd_help->{$command}) {
1278     print "ERROR: unknown command '$command'";
1279     usage(1); die "\n";
1280 }
1281
1282 my @arg = @ARGV;
1283 my $param = parse_argv(@arg);
1284
1285 sub check_params {
1286     for (@_) {
1287         die "$cmd_help->{$command}\n" if !$param->{$_};
1288     }
1289 }
1290
1291 if ($command eq 'destroy') {
1292     check_params(qw(source));
1293
1294     check_target($param->{source});
1295     destroy_job($param);
1296
1297 } elsif ($command eq 'sync') {
1298     check_params(qw(source dest));
1299
1300     check_target($param->{source});
1301     check_target($param->{dest});
1302     sync($param);
1303
1304 } elsif ($command eq 'create') {
1305     check_params(qw(source dest));
1306
1307     check_target($param->{source});
1308     check_target($param->{dest});
1309     init($param);
1310
1311 } elsif ($command eq 'status') {
1312     print status();
1313
1314 } elsif ($command eq 'list') {
1315     print list();
1316
1317 } elsif ($command eq 'help') {
1318     my $help_command = $ARGV[1];
1319
1320     if ($help_command && $cmd_help->{$help_command}) {
1321         die "$cmd_help->{$help_command}\n";
1322
1323     }
1324     if ($param->{verbose}) {
1325         exec("man $PROGNAME");
1326
1327     } else {
1328         usage(1);
1329
1330     }
1331
1332 } elsif ($command eq 'enable') {
1333     check_params(qw(source));
1334
1335     check_target($param->{source});
1336     enable_job($param);
1337
1338 } elsif ($command eq 'disable') {
1339     check_params(qw(source));
1340
1341     check_target($param->{source});
1342     disable_job($param);
1343
1344 } elsif ($command eq 'printpod') {
1345     print_pod();
1346 }
1347
1348 sub usage {
1349     my ($help) = @_;
1350
1351     print("ERROR:\tno command specified\n") if !$help;
1352     print("USAGE:\t$PROGNAME <COMMAND> [ARGS] [OPTIONS]\n");
1353     print("\t$PROGNAME help [<cmd>] [OPTIONS]\n\n");
1354     print("\t$PROGNAME create --dest <string> --source <string> [OPTIONS]\n");
1355     print("\t$PROGNAME destroy --source <string> [OPTIONS]\n");
1356     print("\t$PROGNAME disable --source <string> [OPTIONS]\n");
1357     print("\t$PROGNAME enable --source <string> [OPTIONS]\n");
1358     print("\t$PROGNAME list\n");
1359     print("\t$PROGNAME status\n");
1360     print("\t$PROGNAME sync --dest <string> --source <string> [OPTIONS]\n");
1361 }
1362
1363 sub check_target {
1364     my ($target) = @_;
1365     parse_target($target);
1366 }
1367
1368 sub print_pod {
1369
1370     my $synopsis = join("\n", sort values %$cmd_help);
1371     my $commands = join(", ", sort keys %$cmd_help);
1372
1373     print <<EOF;
1374 =head1 NAME
1375
1376 pve-zsync - PVE ZFS Storage Sync Tool
1377
1378 =head1 SYNOPSIS
1379
1380 pve-zsync <COMMAND> [ARGS] [OPTIONS]
1381
1382 Where <COMMAND> can be one of: $commands
1383
1384 =head1 DESCRIPTION
1385
1386 The pve-zsync tool can help you to sync your VMs or directories stored on ZFS
1387 between multiple servers.
1388
1389 pve-zsync is able to automatically configure CRON jobs, so that a periodic sync
1390 will be automatically triggered.
1391 The default sync interval is 15 min, if you want to change this value you can
1392 do this in F</etc/cron.d/pve-zsync>. If you need help to configure CRON tabs, see
1393 man crontab.
1394
1395 =head1 COMMANDS AND OPTIONS
1396
1397 $synopsis
1398
1399 =head1 EXAMPLES
1400
1401 Adds a job for syncing the local VM 100 to a remote server's ZFS pool named "tank":
1402     pve-zsync create --source=100 -dest=192.168.1.2:tank
1403
1404 =head1 IMPORTANT FILES
1405
1406 Cron jobs and config are stored in F</etc/cron.d/pve-zsync>
1407
1408 The VM configuration itself gets copied to the destination machines
1409 F</var/lib/pve-zsync/> path.
1410
1411 =head1 COPYRIGHT AND DISCLAIMER
1412
1413 Copyright (C) 2007-2021 Proxmox Server Solutions GmbH
1414
1415 This program is free software: you can redistribute it and/or modify it under
1416 the terms of the GNU Affero General Public License as published by the Free
1417 Software Foundation, either version 3 of the License, or (at your option) any
1418 later version.
1419
1420 This program is distributed in the hope that it will be useful, but WITHOUT ANY
1421 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
1422 PARTICULAR PURPOSE. See the GNU Affero General Public License for more
1423 details.
1424
1425 You should have received a copy of the GNU Affero General Public License along
1426 with this program. If not, see <http://www.gnu.org/licenses/>.
1427
1428 EOF
1429 }