pve-zsync

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl qw(:flock SEEK_END);
   7 use Getopt::Long qw(GetOptionsFromArray);
   8 use File::Path qw(make_path);
   9 use JSON;
  10 use IO::File;
  11 use String::ShellQuote 'shell_quote';
  12
  13 my $PROGNAME = "pve-zsync";
  14 my $CONFIG_PATH = "/var/lib/${PROGNAME}";
  15 my $STATE = "${CONFIG_PATH}/sync_state";
  16 my $CRONJOBS = "/etc/cron.d/$PROGNAME";
  17 my $PATH = "/usr/sbin";
  18 my $PVE_DIR = "/etc/pve/local";
  19 my $QEMU_CONF = "${PVE_DIR}/qemu-server";
  20 my $LXC_CONF = "${PVE_DIR}/lxc";
  21 my $PROG_PATH = "$PATH/${PROGNAME}";
  22 my $INTERVAL = 15;
  23 my $DEBUG;
  24
  25 BEGIN {
  26     $DEBUG = 0; # change default here. not above on declaration!
  27     $DEBUG ||= $ENV{ZSYNC_DEBUG};
  28     if ($DEBUG) {
  29         require Data::Dumper;
  30         Data::Dumper->import();
  31     }
  32 }
  33
  34 my $IPV4OCTET = "(?:25[0-5]|(?:[1-9]|1[0-9]|2[0-4])?[0-9])";
  35 my $IPV4RE = "(?:(?:$IPV4OCTET\\.){3}$IPV4OCTET)";
  36 my $IPV6H16 = "(?:[0-9a-fA-F]{1,4})";
  37 my $IPV6LS32 = "(?:(?:$IPV4RE|$IPV6H16:$IPV6H16))";
  38
  39 my $IPV6RE = "(?:" .
  40     "(?:(?:" .                             "(?:$IPV6H16:){6})$IPV6LS32)|" .
  41     "(?:(?:" .                           "::(?:$IPV6H16:){5})$IPV6LS32)|" .
  42     "(?:(?:(?:" .              "$IPV6H16)?::(?:$IPV6H16:){4})$IPV6LS32)|" .
  43     "(?:(?:(?:(?:$IPV6H16:){0,1}$IPV6H16)?::(?:$IPV6H16:){3})$IPV6LS32)|" .
  44     "(?:(?:(?:(?:$IPV6H16:){0,2}$IPV6H16)?::(?:$IPV6H16:){2})$IPV6LS32)|" .
  45     "(?:(?:(?:(?:$IPV6H16:){0,3}$IPV6H16)?::(?:$IPV6H16:){1})$IPV6LS32)|" .
  46     "(?:(?:(?:(?:$IPV6H16:){0,4}$IPV6H16)?::" .           ")$IPV6LS32)|" .
  47     "(?:(?:(?:(?:$IPV6H16:){0,5}$IPV6H16)?::" .            ")$IPV6H16)|" .
  48     "(?:(?:(?:(?:$IPV6H16:){0,6}$IPV6H16)?::" .                    ")))";
  49
  50 my $HOSTv4RE0 = "(?:[\\w\\.\\-_]+|$IPV4RE)";       # hostname or ipv4 address
  51 my $HOSTv4RE1 = "(?:$HOSTv4RE0|\\[$HOSTv4RE0\\])"; # these may be in brackets, too
  52 my $HOSTRE = "(?:$HOSTv4RE1|\\[$IPV6RE\\])";       # ipv6 must always be in brackets
  53 # targets are either a VMID, or a 'host:zpool/path' with 'host:' being optional
  54 my $TARGETRE = qr!^(?:($HOSTRE):)?(\d+|(?:[\w\-_]+)(/.+)?)$!;
  55
  56 my $DISK_KEY_RE = qr/^(?:(?:(?:virtio|ide|scsi|sata|efidisk|mp)\d+)|rootfs): /;
  57
  58 my $INSTANCE_ID = get_instance_id($$);
  59
  60 my $command = $ARGV[0];
  61
  62 if (defined($command) && $command ne 'help' && $command ne 'printpod') {
  63     check_bin ('cstream');
  64     check_bin ('zfs');
  65     check_bin ('ssh');
  66     check_bin ('scp');
  67 }
  68
  69 $SIG{TERM} = $SIG{QUIT} = $SIG{PIPE} = $SIG{HUP} = $SIG{KILL} = $SIG{INT} =  sub {
  70     die "Signaled, aborting sync: $!\n";
  71 };
  72
  73 sub check_bin {
  74     my ($bin)  = @_;
  75
  76     foreach my $p (split (/:/, $ENV{PATH})) {
  77         my $fn = "$p/$bin";
  78         if (-x $fn) {
  79             return $fn;
  80         }
  81     }
  82
  83     die "unable to find command '$bin'\n";
  84 }
  85
  86 sub read_file {
  87     my ($filename, $one_line_only) = @_;
  88
  89     my $fh = IO::File->new($filename, "r")
  90         or die "Could not open file ${filename}: $!\n";
  91
  92     my $text = $one_line_only ? <$fh> : [ <$fh> ];
  93
  94     close($fh);
  95
  96     return $text;
  97 }
  98
  99 sub cut_target_width {
 100     my ($path, $maxlen) = @_;
 101     $path =~ s@/+@/@g;
 102
 103     return $path if length($path) <= $maxlen;
 104
 105     return '..'.substr($path, -$maxlen+2) if $path !~ m@/@;
 106
 107     $path =~ s@/([^/]+/?)$@@;
 108     my $tail = $1;
 109
 110     if (length($tail)+3 == $maxlen) {
 111         return "../$tail";
 112     } elsif (length($tail)+2 >= $maxlen) {
 113         return '..'.substr($tail, -$maxlen+2)
 114     }
 115
 116     $path =~ s@(/[^/]+)(?:/|$)@@;
 117     my $head = $1;
 118     my $both = length($head) + length($tail);
 119     my $remaining = $maxlen-$both-4; # -4 for "/../"
 120
 121     if ($remaining < 0) {
 122         return substr($head, 0, $maxlen - length($tail) - 3) . "../$tail"; # -3 for "../"
 123     }
 124
 125     substr($path, ($remaining/2), (length($path)-$remaining), '..');
 126     return "$head/" . $path . "/$tail";
 127 }
 128
 129 sub locked {
 130     my ($lock_fn, $code) = @_;
 131
 132     my $lock_fh = IO::File->new("> $lock_fn");
 133
 134     flock($lock_fh, LOCK_EX) || die "Couldn't acquire lock - $!\n";
 135     my $res = eval { $code->() };
 136     my $err = $@;
 137
 138     flock($lock_fh, LOCK_UN) || warn "Error unlocking - $!\n";
 139     die "$err" if $err;
 140
 141     close($lock_fh);
 142     return $res;
 143 }
 144
 145 sub get_status {
 146     my ($source, $name, $status) = @_;
 147
 148     if ($status->{$source->{all}}->{$name}->{status}) {
 149         return $status;
 150     }
 151
 152     return undef;
 153 }
 154
 155 sub check_dataset_exists {
 156     my ($dataset, $ip, $user) = @_;
 157
 158     my $cmd = [];
 159
 160     if ($ip) {
 161         push @$cmd, 'ssh', "$user\@$ip", '--';
 162     }
 163     push @$cmd, 'zfs', 'list', '-H', '--', $dataset;
 164     eval {
 165         run_cmd($cmd);
 166     };
 167
 168     if ($@) {
 169         return 0;
 170     }
 171     return 1;
 172 }
 173
 174 sub create_file_system {
 175     my ($file_system, $ip, $user) = @_;
 176
 177     my $cmd = [];
 178
 179     if ($ip) {
 180         push @$cmd, 'ssh', "$user\@$ip", '--';
 181     }
 182     push @$cmd, 'zfs', 'create', $file_system;
 183
 184     run_cmd($cmd);
 185 }
 186
 187 sub parse_target {
 188     my ($text) = @_;
 189
 190     my $errstr = "$text : is not a valid input! Use [IP:]<VMID> or [IP:]<ZFSPool>[/Path]";
 191     my $target = {};
 192
 193     if ($text !~ $TARGETRE) {
 194         die "$errstr\n";
 195     }
 196     $target->{all} = $2;
 197     $target->{ip} = $1 if $1;
 198     my @parts = split('/', $2);
 199
 200     $target->{ip} =~ s/^\[(.*)\]$/$1/ if $target->{ip};
 201
 202     my $pool = $target->{pool} = shift(@parts);
 203     die "$errstr\n" if !$pool;
 204
 205     if ($pool =~ m/^\d+$/) {
 206         $target->{vmid} = $pool;
 207         delete $target->{pool};
 208     }
 209
 210     return $target if (@parts == 0);
 211     $target->{last_part} = pop(@parts);
 212
 213     if ($target->{ip}) {
 214         pop(@parts);
 215     }
 216     if (@parts > 0) {
 217         $target->{path} = join('/', @parts);
 218     }
 219
 220     return $target;
 221 }
 222
 223 sub read_cron {
 224
 225     #This is for the first use to init file;
 226     if (!-e $CRONJOBS) {
 227         my $new_fh = IO::File->new("> $CRONJOBS");
 228         die "Could not create $CRONJOBS: $!\n" if !$new_fh;
 229         close($new_fh);
 230         return undef;
 231     }
 232
 233     my $text = read_file($CRONJOBS, 0);
 234
 235     return encode_cron(@{$text});
 236 }
 237
 238 sub parse_argv {
 239     my (@arg) = @_;
 240
 241     my $param = {
 242         dest => undef,
 243         source => undef,
 244         verbose => undef,
 245         limit => undef,
 246         maxsnap => undef,
 247         name => undef,
 248         skip => undef,
 249         method => undef,
 250         source_user => undef,
 251         dest_user => undef,
 252         prepend_storage_id => undef,
 253         properties => undef,
 254         dest_config_path => undef,
 255     };
 256
 257     my ($ret) = GetOptionsFromArray(
 258         \@arg,
 259         'dest=s' => \$param->{dest},
 260         'source=s' => \$param->{source},
 261         'verbose' => \$param->{verbose},
 262         'limit=i' => \$param->{limit},
 263         'maxsnap=i' => \$param->{maxsnap},
 264         'name=s' => \$param->{name},
 265         'skip' => \$param->{skip},
 266         'method=s' => \$param->{method},
 267         'source-user=s' => \$param->{source_user},
 268         'dest-user=s' => \$param->{dest_user},
 269         'prepend-storage-id' => \$param->{prepend_storage_id},
 270         'properties' => \$param->{properties},
 271         'dest-config-path=s' => \$param->{dest_config_path},
 272     );
 273
 274     die "can't parse options\n" if $ret == 0;
 275
 276     $param->{name} //= "default";
 277     $param->{maxsnap} //= 1;
 278     $param->{method} //= "ssh";
 279     $param->{source_user} //= "root";
 280     $param->{dest_user} //= "root";
 281
 282     return $param;
 283 }
 284
 285 sub add_state_to_job {
 286     my ($job) = @_;
 287
 288     my $states = read_state();
 289     my $state = $states->{$job->{source}}->{$job->{name}};
 290
 291     $job->{state} = $state->{state};
 292     $job->{lsync} = $state->{lsync};
 293     $job->{vm_type} = $state->{vm_type};
 294     $job->{instance_id} = $state->{instance_id};
 295
 296     for (my $i = 0; $state->{"snap$i"}; $i++) {
 297         $job->{"snap$i"} = $state->{"snap$i"};
 298     }
 299
 300     return $job;
 301 }
 302
 303 sub encode_cron {
 304     my (@text) = @_;
 305
 306     my $cfg = {};
 307
 308     while (my $line = shift(@text)) {
 309
 310         my @arg = split('\s', $line);
 311         my $param = parse_argv(@arg);
 312
 313         if ($param->{source} && $param->{dest}) {
 314             my $source = delete $param->{source};
 315             my $name = delete $param->{name};
 316
 317             $cfg->{$source}->{$name} = $param;
 318         }
 319     }
 320
 321     return $cfg;
 322 }
 323
 324 sub param_to_job {
 325     my ($param) = @_;
 326
 327     my $job = {};
 328
 329     my $source = parse_target($param->{source});
 330     my $dest;
 331     $dest = parse_target($param->{dest}) if $param->{dest};
 332
 333     $job->{name} = !$param->{name} ? "default" : $param->{name};
 334     $job->{dest} = $param->{dest} if $param->{dest};
 335     $job->{method} = "local" if !$dest->{ip} && !$source->{ip};
 336     $job->{method} = "ssh" if !$job->{method};
 337     $job->{limit} = $param->{limit};
 338     $job->{maxsnap} = $param->{maxsnap};
 339     $job->{source} = $param->{source};
 340     $job->{source_user} = $param->{source_user};
 341     $job->{dest_user} = $param->{dest_user};
 342     $job->{prepend_storage_id} = !!$param->{prepend_storage_id};
 343     $job->{properties} = !!$param->{properties};
 344     $job->{dest_config_path} = $param->{dest_config_path} if $param->{dest_config_path};
 345
 346     return $job;
 347 }
 348
 349 sub read_state {
 350
 351     if (!-e $STATE) {
 352         make_path $CONFIG_PATH;
 353         my $new_fh = IO::File->new("> $STATE");
 354         die "Could not create $STATE: $!\n" if !$new_fh;
 355         print $new_fh "{}";
 356         close($new_fh);
 357         return undef;
 358     }
 359
 360     my $text = read_file($STATE, 1);
 361     return decode_json($text);
 362 }
 363
 364 sub update_state {
 365     my ($job) = @_;
 366
 367     my $text = eval { read_file($STATE, 1); };
 368
 369     my $out_fh = IO::File->new("> $STATE.new");
 370     die "Could not open file ${STATE}.new: $!\n" if !$out_fh;
 371
 372     my $states = {};
 373     my $state = {};
 374     if ($text){
 375         $states = decode_json($text);
 376         $state = $states->{$job->{source}}->{$job->{name}};
 377     }
 378
 379     if ($job->{state} ne "del") {
 380         $state->{state} = $job->{state};
 381         $state->{lsync} = $job->{lsync};
 382         $state->{instance_id} = $job->{instance_id};
 383         $state->{vm_type} = $job->{vm_type};
 384
 385         for (my $i = 0; $job->{"snap$i"} ; $i++) {
 386             $state->{"snap$i"} = $job->{"snap$i"};
 387         }
 388         $states->{$job->{source}}->{$job->{name}} = $state;
 389     } else {
 390
 391         delete $states->{$job->{source}}->{$job->{name}};
 392         delete $states->{$job->{source}} if !keys %{$states->{$job->{source}}};
 393     }
 394
 395     $text = encode_json($states);
 396     print $out_fh $text;
 397
 398     close($out_fh);
 399     rename "$STATE.new", $STATE;
 400
 401     return $states;
 402 }
 403
 404 sub update_cron {
 405     my ($job) = @_;
 406
 407     my $updated;
 408     my $has_header;
 409     my $line_no = 0;
 410     my $text = "";
 411     my $header = "SHELL=/bin/sh\n";
 412     $header .= "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n\n";
 413
 414     my $current = read_file($CRONJOBS, 0);
 415
 416     foreach my $line (@{$current}) {
 417         chomp($line);
 418         if ($line =~ m/source $job->{source} .*name $job->{name} /) {
 419             $updated = 1;
 420             next if $job->{state} eq "del";
 421             $text .= format_job($job, $line);
 422         } else {
 423             if (($line_no < 3) && ($line =~ /^(PATH|SHELL)/ )) {
 424                 $has_header = 1;
 425             }
 426             $text .= "$line\n";
 427         }
 428         $line_no++;
 429     }
 430
 431     if (!$has_header) {
 432         $text = "$header$text";
 433     }
 434
 435     if (!$updated) {
 436         $text .= format_job($job);
 437     }
 438     my $new_fh = IO::File->new("> ${CRONJOBS}.new");
 439     die "Could not open file ${CRONJOBS}.new: $!\n" if !$new_fh;
 440
 441     print $new_fh $text or die "can't write to $CRONJOBS.new: $!\n";
 442     close ($new_fh);
 443
 444     rename "${CRONJOBS}.new", $CRONJOBS or die "can't move $CRONJOBS.new: $!\n";
 445 }
 446
 447 sub format_job {
 448     my ($job, $line) = @_;
 449     my $text = "";
 450
 451     if ($job->{state} eq "stopped") {
 452         $text = "#";
 453     }
 454     if ($line) {
 455         $line =~ /^#*\s*((?:\S+\s+){4}\S+)\s+root/;
 456         $text .= $1;
 457     } else {
 458         $text .= "*/$INTERVAL * * * *";
 459     }
 460     $text .= " root";
 461     $text .= " $PROGNAME sync --source $job->{source} --dest $job->{dest}";
 462     $text .= " --name $job->{name} --maxsnap $job->{maxsnap}";
 463     $text .= " --limit $job->{limit}" if $job->{limit};
 464     $text .= " --method $job->{method}";
 465     $text .= " --verbose" if $job->{verbose};
 466     $text .= " --source-user $job->{source_user}";
 467     $text .= " --dest-user $job->{dest_user}";
 468     $text .= " --prepend-storage-id" if $job->{prepend_storage_id};
 469     $text .= " --properties" if $job->{properties};
 470     $text .= " --dest-config-path $job->{dest_config_path}" if $job->{dest_config_path};
 471     $text .= "\n";
 472
 473     return $text;
 474 }
 475
 476 sub list {
 477
 478     my $cfg = read_cron();
 479
 480     my $list = sprintf("%-25s%-25s%-10s%-20s%-6s%-5s\n" , "SOURCE", "NAME", "STATE", "LAST SYNC", "TYPE", "CON");
 481
 482     my $states = read_state();
 483     foreach my $source (sort keys%{$cfg}) {
 484         foreach my $name (sort keys%{$cfg->{$source}}) {
 485             $list .= sprintf("%-25s", cut_target_width($source, 25));
 486             $list .= sprintf("%-25s", cut_target_width($name, 25));
 487             $list .= sprintf("%-10s", $states->{$source}->{$name}->{state});
 488             $list .= sprintf("%-20s", $states->{$source}->{$name}->{lsync});
 489             $list .= sprintf("%-6s", defined($states->{$source}->{$name}->{vm_type}) ? $states->{$source}->{$name}->{vm_type} : "undef");
 490             $list .= sprintf("%-5s\n", $cfg->{$source}->{$name}->{method});
 491         }
 492     }
 493
 494     return $list;
 495 }
 496
 497 sub vm_exists {
 498     my ($target, $user) = @_;
 499
 500     return undef if !defined($target->{vmid});
 501
 502     my $conf_fn = "$target->{vmid}.conf";
 503
 504     if ($target->{ip}) {
 505         my @cmd = ('ssh', "$user\@$target->{ip}", '--', '/bin/ls');
 506         return "qemu" if eval { run_cmd([@cmd, "$QEMU_CONF/$conf_fn"]) };
 507         return "lxc" if  eval { run_cmd([@cmd, "$LXC_CONF/$conf_fn"]) };
 508     } else {
 509         return "qemu" if -f "$QEMU_CONF/$conf_fn";
 510         return "lxc" if -f "$LXC_CONF/$conf_fn";
 511     }
 512
 513     return undef;
 514 }
 515
 516 sub init {
 517     my ($param) = @_;
 518
 519     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 520         my $cfg = read_cron();
 521
 522         my $job = param_to_job($param);
 523
 524         $job->{state} = "ok";
 525         $job->{lsync} = 0;
 526
 527         my $source = parse_target($param->{source});
 528         my $dest = parse_target($param->{dest});
 529
 530         if (my $ip =  $dest->{ip}) {
 531             run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{dest_user}\@$ip"]);
 532         }
 533
 534         if (my $ip =  $source->{ip}) {
 535             run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{source_user}\@$ip"]);
 536         }
 537
 538         die "Pool $dest->{all} does not exist\n"
 539             if !check_dataset_exists($dest->{all}, $dest->{ip}, $param->{dest_user});
 540
 541         if (!defined($source->{vmid})) {
 542             die "Pool $source->{all} does not exist\n"
 543                 if !check_dataset_exists($source->{all}, $source->{ip}, $param->{source_user});
 544         }
 545
 546         my $vm_type = vm_exists($source, $param->{source_user});
 547         $job->{vm_type} = $vm_type;
 548         $source->{vm_type} = $vm_type;
 549
 550         die "VM $source->{vmid} doesn't exist\n" if $source->{vmid} && !$vm_type;
 551
 552         die "Config already exists\n" if $cfg->{$job->{source}}->{$job->{name}};
 553
 554         #check if vm has zfs disks if not die;
 555         get_disks($source, $param->{source_user}) if $source->{vmid};
 556
 557         update_cron($job);
 558         update_state($job);
 559     }); #cron and state lock
 560
 561     return if $param->{skip};
 562
 563     eval { sync($param) };
 564     if (my $err = $@) {
 565         destroy_job($param);
 566         print $err;
 567     }
 568 }
 569
 570 sub get_job {
 571     my ($param) = @_;
 572
 573     my $cfg = read_cron();
 574
 575     if (!$cfg->{$param->{source}}->{$param->{name}}) {
 576         die "Job  with source $param->{source} and name $param->{name} does not exist\n" ;
 577     }
 578     my $job = $cfg->{$param->{source}}->{$param->{name}};
 579     $job->{name} = $param->{name};
 580     $job->{source} = $param->{source};
 581     $job = add_state_to_job($job);
 582
 583     return $job;
 584 }
 585
 586 sub destroy_job {
 587     my ($param) = @_;
 588
 589     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 590         my $job = get_job($param);
 591         $job->{state} = "del";
 592
 593         update_cron($job);
 594         update_state($job);
 595     });
 596 }
 597
 598 sub get_instance_id {
 599     my ($pid) = @_;
 600
 601     my $stat = read_file("/proc/$pid/stat", 1)
 602         or die "unable to read process stats\n";
 603     my $boot_id = read_file("/proc/sys/kernel/random/boot_id", 1)
 604         or die "unable to read boot ID\n";
 605
 606     my $stats = [ split(/\s+/, $stat) ];
 607     my $starttime = $stats->[21];
 608     chomp($boot_id);
 609
 610     return "${pid}:${starttime}:${boot_id}";
 611 }
 612
 613 sub instance_exists {
 614     my ($instance_id) = @_;
 615
 616     if (defined($instance_id) && $instance_id =~ m/^([1-9][0-9]*):/) {
 617         my $pid = $1;
 618         my $actual_id = eval { get_instance_id($pid); };
 619         return defined($actual_id) && $actual_id eq $instance_id;
 620     }
 621
 622     return 0;
 623 }
 624
 625 sub sync {
 626     my ($param) = @_;
 627
 628     my $job;
 629
 630     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 631         eval { $job = get_job($param) };
 632
 633         if ($job) {
 634             my $state = $job->{state} // 'ok';
 635             $state = 'ok' if !instance_exists($job->{instance_id});
 636
 637             if ($state eq "syncing" || $state eq "waiting") {
 638                 die "Job --source $param->{source} --name $param->{name} is already scheduled to sync\n";
 639             }
 640
 641             $job->{state} = "waiting";
 642             $job->{instance_id} = $INSTANCE_ID;
 643
 644             update_state($job);
 645         }
 646     });
 647
 648     locked("$CONFIG_PATH/sync.lock", sub {
 649
 650         my $date = get_date();
 651
 652         my $dest;
 653         my $source;
 654         my $vm_type;
 655
 656         locked("$CONFIG_PATH/cron_and_state.lock", sub {
 657             #job might've changed while we waited for the sync lock, but we can be sure it's not syncing
 658             eval { $job = get_job($param); };
 659
 660             if ($job && defined($job->{state}) && $job->{state} eq "stopped") {
 661                 die "Job --source $param->{source} --name $param->{name} has been disabled\n";
 662             }
 663
 664             $dest = parse_target($param->{dest});
 665             $source = parse_target($param->{source});
 666
 667             $vm_type = vm_exists($source, $param->{source_user});
 668             $source->{vm_type} = $vm_type;
 669
 670             if ($job) {
 671                 $job->{state} = "syncing";
 672                 $job->{vm_type} = $vm_type if !$job->{vm_type};
 673                 update_state($job);
 674             }
 675         }); #cron and state lock
 676
 677         my $sync_path = sub {
 678             my ($source, $dest, $job, $param, $date) = @_;
 679
 680             my $dest_dataset = target_dataset($source, $dest);
 681
 682             ($dest->{old_snap}, $dest->{last_snap}) = snapshot_get(
 683                 $dest_dataset,
 684                 $param->{maxsnap},
 685                 $param->{name},
 686                 $dest->{ip},
 687                 $param->{dest_user},
 688             );
 689
 690             prepare_prepended_target($source, $dest, $param->{dest_user}) if defined($dest->{prepend});
 691
 692             snapshot_add($source, $dest, $param->{name}, $date, $param->{source_user}, $param->{dest_user});
 693
 694             send_image($source, $dest, $param);
 695
 696             for my $old_snap (@{$dest->{old_snap}}) {
 697                 snapshot_destroy($source, $dest, $param->{method}, $old_snap, $param->{source_user}, $param->{dest_user});
 698             }
 699         };
 700
 701         eval{
 702             if ($source->{vmid}) {
 703                 die "VM $source->{vmid} doesn't exist\n" if !$vm_type;
 704                 die "source-user has to be root for syncing VMs\n" if ($param->{source_user} ne "root");
 705                 my $disks = get_disks($source, $param->{source_user});
 706
 707                 foreach my $disk (sort keys %{$disks}) {
 708                     $source->{all} = $disks->{$disk}->{all};
 709                     $source->{pool} = $disks->{$disk}->{pool};
 710                     $source->{path} = $disks->{$disk}->{path} if $disks->{$disk}->{path};
 711                     $source->{last_part} = $disks->{$disk}->{last_part};
 712
 713                     $dest->{prepend} = $disks->{$disk}->{storage_id}
 714                         if $param->{prepend_storage_id};
 715
 716                     &$sync_path($source, $dest, $job, $param, $date);
 717                 }
 718                 if ($param->{method} eq "ssh" && ($source->{ip} || $dest->{ip})) {
 719                     send_config($source, $dest,'ssh', $param->{source_user}, $param->{dest_user}, $param->{dest_config_path});
 720                 } else {
 721                     send_config($source, $dest,'local', $param->{source_user}, $param->{dest_user}, $param->{dest_config_path});
 722                 }
 723             } else {
 724                 &$sync_path($source, $dest, $job, $param, $date);
 725             }
 726         };
 727         if (my $err = $@) {
 728             locked("$CONFIG_PATH/cron_and_state.lock", sub {
 729                 eval { $job = get_job($param); };
 730                 if ($job) {
 731                     $job->{state} = "error";
 732                     delete $job->{instance_id};
 733                     update_state($job);
 734                 }
 735             });
 736             print "Job --source $param->{source} --name $param->{name} got an ERROR!!!\nERROR Message:\n";
 737             die "$err\n";
 738         }
 739
 740         locked("$CONFIG_PATH/cron_and_state.lock", sub {
 741             eval { $job = get_job($param); };
 742             if ($job) {
 743                 if (defined($job->{state}) && $job->{state} eq "stopped") {
 744                     $job->{state} = "stopped";
 745                 } else {
 746                     $job->{state} = "ok";
 747                 }
 748                 $job->{lsync} = $date;
 749                 delete $job->{instance_id};
 750                 update_state($job);
 751             }
 752         });
 753     }); #sync lock
 754 }
 755
 756 sub snapshot_get{
 757     my ($dataset, $max_snap, $name, $ip, $user) = @_;
 758
 759     my $cmd = [];
 760     push @$cmd, 'ssh', "$user\@$ip", '--', if $ip;
 761     push @$cmd, 'zfs', 'list', '-r', '-t', 'snapshot', '-Ho', 'name', '-S', 'creation';
 762     push @$cmd, $dataset;
 763
 764     my $raw;
 765     eval {$raw = run_cmd($cmd)};
 766     if (my $erro =$@) { #this means the volume doesn't exist on dest yet
 767        return undef;
 768     }
 769
 770     my $index = 0;
 771     my $line = "";
 772     my $last_snap = undef;
 773     my $old_snap = [];
 774
 775     while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
 776         $line = $1;
 777         if ($line =~ m/@(.*)$/) {
 778             $last_snap = $1 if (!$last_snap);
 779         }
 780         if ($line =~ m/(rep_\Q${name}\E_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})$/) {
 781             # interpreted as infinity
 782             last if $max_snap <= 0;
 783
 784             my $snap = $1;
 785             $index++;
 786
 787             if ($index >= $max_snap) {
 788                 push @{$old_snap}, $snap;
 789             }
 790         }
 791     }
 792
 793     return ($old_snap, $last_snap) if $last_snap;
 794
 795     return undef;
 796 }
 797
 798 sub snapshot_add {
 799     my ($source, $dest, $name, $date, $source_user, $dest_user) = @_;
 800
 801     my $snap_name = "rep_$name\_".$date;
 802
 803     $source->{new_snap} = $snap_name;
 804
 805     my $path = "$source->{all}\@$snap_name";
 806
 807     my $cmd = [];
 808     push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--', if $source->{ip};
 809     push @$cmd, 'zfs', 'snapshot', $path;
 810     eval{
 811         run_cmd($cmd);
 812     };
 813
 814     if (my $err = $@) {
 815         snapshot_destroy($source, $dest, 'ssh', $snap_name, $source_user, $dest_user);
 816         die "$err\n";
 817     }
 818 }
 819
 820 sub get_disks {
 821     my ($target, $user) = @_;
 822
 823     my $cmd = [];
 824     push @$cmd, 'ssh', "$user\@$target->{ip}", '--', if $target->{ip};
 825
 826     if ($target->{vm_type} eq 'qemu') {
 827         push @$cmd, 'qm', 'config', $target->{vmid};
 828     } elsif ($target->{vm_type} eq 'lxc') {
 829         push @$cmd, 'pct', 'config', $target->{vmid};
 830     } else {
 831         die "VM Type unknown\n";
 832     }
 833
 834     my $res = run_cmd($cmd);
 835
 836     my $disks = parse_disks($res, $target->{ip}, $target->{vm_type}, $user);
 837
 838     return $disks;
 839 }
 840
 841 sub run_cmd {
 842     my ($cmd) = @_;
 843     print "Start CMD\n" if $DEBUG;
 844     print Dumper $cmd if $DEBUG;
 845     if (ref($cmd) eq 'ARRAY') {
 846         $cmd = join(' ', map { ref($_) ? $$_ : shell_quote($_) } @$cmd);
 847     }
 848     my $output = `$cmd 2>&1`;
 849
 850     die "COMMAND:\n\t$cmd\nGET ERROR:\n\t$output" if 0 != $?;
 851
 852     chomp($output);
 853     print Dumper $output if $DEBUG;
 854     print "END CMD\n" if $DEBUG;
 855     return $output;
 856 }
 857
 858 sub parse_disks {
 859     my ($text, $ip, $vm_type, $user) = @_;
 860
 861     my $disks;
 862
 863     my $num = 0;
 864     while ($text && $text =~ s/^(.*?)(\n|$)//) {
 865         my $line = $1;
 866
 867         next if $line =~ /media=cdrom/;
 868         next if $line !~ m/$DISK_KEY_RE/;
 869
 870         #QEMU if backup is not set include in  sync
 871         next if $vm_type eq 'qemu' && ($line =~ m/backup=(?i:0|no|off|false)/);
 872
 873         #LXC if backup is not set do no in sync
 874         next if $vm_type eq 'lxc' && ($line =~ m/^mp\d:/) && ($line !~ m/backup=(?i:1|yes|on|true)/);
 875
 876         my $disk = undef;
 877         my $stor = undef;
 878         if($line =~ m/$DISK_KEY_RE(.*)$/) {
 879             my @parameter = split(/,/,$1);
 880
 881             foreach my $opt (@parameter) {
 882                 if ($opt =~ m/^(?:file=|volume=)?([^:]+):([A-Za-z0-9\-]+)$/){
 883                     $disk = $2;
 884                     $stor = $1;
 885                     last;
 886                 }
 887             }
 888         }
 889         if (!defined($disk) || !defined($stor)) {
 890             print "Disk: \"$line\" has no valid zfs dataset format and will be skipped\n";
 891             next;
 892         }
 893
 894         my $cmd = [];
 895         push @$cmd, 'ssh', "$user\@$ip", '--' if $ip;
 896         push @$cmd, 'pvesm', 'path', "$stor:$disk";
 897         my $path = run_cmd($cmd);
 898
 899         die "Get no path from pvesm path $stor:$disk\n" if !$path;
 900
 901         $disks->{$num}->{storage_id} = $stor;
 902
 903         if ($vm_type eq 'qemu' && $path =~ m/^\/dev\/zvol\/(\w+.*)(\/$disk)$/) {
 904
 905             my @array = split('/', $1);
 906             $disks->{$num}->{pool} = shift(@array);
 907             $disks->{$num}->{all} = $disks->{$num}->{pool};
 908             if (0 < @array) {
 909                 $disks->{$num}->{path} = join('/', @array);
 910                 $disks->{$num}->{all} .= "\/$disks->{$num}->{path}";
 911             }
 912             $disks->{$num}->{last_part} = $disk;
 913             $disks->{$num}->{all} .= "\/$disk";
 914
 915             $num++;
 916         } elsif ($vm_type eq 'lxc' && $path =~ m/^\/(\w+.+)(\/(\w+.*))*(\/$disk)$/) {
 917
 918             $disks->{$num}->{pool} = $1;
 919             $disks->{$num}->{all} = $disks->{$num}->{pool};
 920
 921             if ($2) {
 922                 $disks->{$num}->{path} = $3;
 923                 $disks->{$num}->{all} .= "\/$disks->{$num}->{path}";
 924             }
 925
 926             $disks->{$num}->{last_part} = $disk;
 927             $disks->{$num}->{all} .= "\/$disk";
 928
 929             $num++;
 930
 931         } else {
 932             die "ERROR: in path\n";
 933         }
 934     }
 935
 936     die "Vm include no disk on zfs.\n" if !$disks->{0};
 937     return $disks;
 938 }
 939
 940 # how the corresponding dataset is named on the target
 941 sub target_dataset {
 942     my ($source, $dest) = @_;
 943
 944     my $target = "$dest->{all}";
 945     $target .= "/$dest->{prepend}" if defined($dest->{prepend});
 946     $target .= "/$source->{last_part}" if $source->{last_part};
 947     $target =~ s!/+!/!g;
 948
 949     return $target;
 950 }
 951
 952 # create the parent dataset for the actual target
 953 sub prepare_prepended_target {
 954     my ($source, $dest, $dest_user) = @_;
 955
 956     die "internal error - not a prepended target\n" if !defined($dest->{prepend});
 957
 958     # The parent dataset shouldn't be the actual target.
 959     die "internal error - no last_part for source\n" if !$source->{last_part};
 960
 961     my $target = "$dest->{all}/$dest->{prepend}";
 962     $target =~ s!/+!/!g;
 963
 964     return if check_dataset_exists($target, $dest->{ip}, $dest_user);
 965
 966     create_file_system($target, $dest->{ip}, $dest_user);
 967 }
 968
 969 sub snapshot_destroy {
 970     my ($source, $dest, $method, $snap, $source_user, $dest_user) = @_;
 971
 972     my @zfscmd = ('zfs', 'destroy');
 973     my $snapshot = "$source->{all}\@$snap";
 974
 975     eval {
 976         if($source->{ip} && $method eq 'ssh'){
 977             run_cmd(['ssh', "$source_user\@$source->{ip}", '--', @zfscmd, $snapshot]);
 978         } else {
 979             run_cmd([@zfscmd, $snapshot]);
 980         }
 981     };
 982     if (my $erro = $@) {
 983         warn "WARN: $erro";
 984     }
 985     if ($dest) {
 986         my @ssh = $dest->{ip} ? ('ssh', "$dest_user\@$dest->{ip}", '--') : ();
 987
 988         my $path = target_dataset($source, $dest);
 989
 990         eval {
 991             run_cmd([@ssh, @zfscmd, "$path\@$snap"]);
 992         };
 993         if (my $erro = $@) {
 994             warn "WARN: $erro";
 995         }
 996     }
 997 }
 998
 999 # check if snapshot for incremental sync exist on source side
1000 sub snapshot_exist {
1001     my ($source , $dest, $method, $source_user) = @_;
1002
1003     my $cmd = [];
1004     push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--' if $source->{ip};
1005     push @$cmd, 'zfs', 'list', '-rt', 'snapshot', '-Ho', 'name';
1006
1007     my $path = $source->{all};
1008     $path .= "\@$dest->{last_snap}";
1009
1010     push @$cmd, $path;
1011
1012     eval {run_cmd($cmd)};
1013     if (my $erro =$@) {
1014         warn "WARN: $erro";
1015         return undef;
1016     }
1017     return 1;
1018 }
1019
1020 sub send_image {
1021     my ($source, $dest, $param) = @_;
1022
1023     my $cmd = [];
1024
1025     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{source_user}\@$source->{ip}", '--' if $source->{ip};
1026     push @$cmd, 'zfs', 'send';
1027     push @$cmd, '-p', if $param->{properties};
1028     push @$cmd, '-v' if $param->{verbose};
1029
1030     if($dest->{last_snap} && snapshot_exist($source , $dest, $param->{method}, $param->{source_user})) {
1031         push @$cmd, '-i', "$source->{all}\@$dest->{last_snap}";
1032     }
1033     push @$cmd, '--', "$source->{all}\@$source->{new_snap}";
1034
1035     if ($param->{limit}){
1036         my $bwl = $param->{limit}*1024;
1037         push @$cmd, \'|', 'cstream', '-t', $bwl;
1038     }
1039     my $target = target_dataset($source, $dest);
1040
1041     push @$cmd, \'|';
1042     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{dest_user}\@$dest->{ip}", '--' if $dest->{ip};
1043     push @$cmd, 'zfs', 'recv', '-F', '--';
1044     push @$cmd, "$target";
1045
1046     eval {
1047         run_cmd($cmd)
1048     };
1049
1050     if (my $erro = $@) {
1051         snapshot_destroy($source, undef, $param->{method}, $source->{new_snap}, $param->{source_user}, $param->{dest_user});
1052         die $erro;
1053     };
1054 }
1055
1056
1057 sub send_config{
1058     my ($source, $dest, $method, $source_user, $dest_user, $dest_config_path) = @_;
1059
1060     my $source_target = $source->{vm_type} eq 'qemu' ? "$QEMU_CONF/$source->{vmid}.conf": "$LXC_CONF/$source->{vmid}.conf";
1061     my $dest_target_new ="$source->{vmid}.conf.$source->{vm_type}.$source->{new_snap}";
1062
1063     my $config_dir = $dest_config_path // $CONFIG_PATH;
1064     $config_dir .= "/$dest->{last_part}" if $dest->{last_part};
1065
1066     $dest_target_new = $config_dir.'/'.$dest_target_new;
1067
1068     if ($method eq 'ssh'){
1069         if ($dest->{ip} && $source->{ip}) {
1070             run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'mkdir', '-p', '--', $config_dir]);
1071             run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", "$dest_user\@[$dest->{ip}]:$dest_target_new"]);
1072         } elsif ($dest->{ip}) {
1073             run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'mkdir', '-p', '--', $config_dir]);
1074             run_cmd(['scp', '--', $source_target, "$dest_user\@[$dest->{ip}]:$dest_target_new"]);
1075         } elsif ($source->{ip}) {
1076             run_cmd(['mkdir', '-p', '--', $config_dir]);
1077             run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", $dest_target_new]);
1078         }
1079
1080         for my $old_snap (@{$dest->{old_snap}}) {
1081             my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.${old_snap}";
1082             if($dest->{ip}){
1083                 run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'rm', '-f', '--', $dest_target_old]);
1084             } else {
1085                 run_cmd(['rm', '-f', '--', $dest_target_old]);
1086             }
1087         }
1088     } elsif ($method eq 'local') {
1089         run_cmd(['mkdir', '-p', '--', $config_dir]);
1090         run_cmd(['cp', $source_target, $dest_target_new]);
1091     }
1092 }
1093
1094 sub get_date {
1095     my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
1096     my $datestamp = sprintf ("%04d-%02d-%02d_%02d:%02d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
1097
1098     return $datestamp;
1099 }
1100
1101 sub status {
1102     my $cfg = read_cron();
1103
1104     my $status_list = sprintf("%-25s%-25s%-10s\n", "SOURCE", "NAME", "STATUS");
1105
1106     my $states = read_state();
1107
1108     foreach my $source (sort keys%{$cfg}) {
1109         foreach my $sync_name (sort keys%{$cfg->{$source}}) {
1110             $status_list .= sprintf("%-25s", cut_target_width($source, 25));
1111             $status_list .= sprintf("%-25s", cut_target_width($sync_name, 25));
1112             $status_list .= "$states->{$source}->{$sync_name}->{state}\n";
1113         }
1114     }
1115
1116     return $status_list;
1117 }
1118
1119 sub enable_job {
1120     my ($param) = @_;
1121
1122     locked("$CONFIG_PATH/cron_and_state.lock", sub {
1123         my $job = get_job($param);
1124         $job->{state} = "ok";
1125         update_state($job);
1126         update_cron($job);
1127     });
1128 }
1129
1130 sub disable_job {
1131     my ($param) = @_;
1132
1133     locked("$CONFIG_PATH/cron_and_state.lock", sub {
1134         my $job = get_job($param);
1135         $job->{state} = "stopped";
1136         update_state($job);
1137         update_cron($job);
1138     });
1139 }
1140
1141 my $cmd_help = {
1142     destroy => qq{
1143 $PROGNAME destroy --source <string> [OPTIONS]
1144
1145     Remove a sync Job from the scheduler
1146
1147         --name      string
1148                 The name of the sync job, if not set 'default' is used.
1149
1150         --source    string
1151                 The source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1152     },
1153     create => qq{
1154 $PROGNAME create --dest <string> --source <string> [OPTIONS]
1155
1156     Create a new sync-job
1157
1158         --dest      string
1159                 The destination target is like [IP]:<Pool>[/Path]
1160
1161         --dest-user string
1162                 The name of the user on the destination target, root by default
1163
1164         --limit     integer
1165                 Maximal sync speed in kBytes/s, default is unlimited
1166
1167         --maxsnap   integer
1168                 The number of snapshots to keep until older ones are erased.
1169                 The default is 1, use 0 for unlimited.
1170
1171         --name      string
1172                 The name of the sync job, if not set it is default
1173
1174         --prepend-storage-id
1175                 If specified, prepend the storage ID to the destination's path(s).
1176
1177         --skip
1178                 If specified, skip the first sync.
1179
1180         --source    string
1181                 The source can be an <VMID> or [IP:]<ZFSPool>[/Path]
1182
1183         --source-user    string
1184                 The (ssh) user-name on the source target, root by default
1185
1186         --properties
1187                 If specified, include the dataset's properties in the stream.
1188
1189         --dest-config-path    string
1190                 Specifies a custom config path on the destination target.
1191                 The default is /var/lib/pve-zsync
1192     },
1193     sync => qq{
1194 $PROGNAME sync --dest <string> --source <string> [OPTIONS]\n
1195
1196     Trigger one sync.
1197
1198         --dest      string
1199                 The destination target is like [IP:]<Pool>[/Path]
1200
1201         --dest-user string
1202                 The (ssh) user-name on the destination target, root by default
1203
1204         --limit     integer
1205                 The maximal sync speed in kBytes/s, default is unlimited
1206
1207         --maxsnap   integer
1208                 The number of snapshots to keep until older ones are erased.
1209                 The default is 1, use 0 for unlimited.
1210
1211         --name      string
1212                 The name of the sync job, if not set it is 'default'.
1213                 It is only necessary if scheduler allready contains this source.
1214
1215         --prepend-storage-id
1216                 If specified, prepend the storage ID to the destination's path(s).
1217
1218         --source    string
1219                 The source can either be an <VMID> or [IP:]<ZFSPool>[/Path]
1220
1221         --source-user    string
1222                 The name of the user on the source target, root by default
1223
1224         --verbose
1225                 If specified, print out the sync progress.
1226
1227         --properties
1228                 If specified, include the dataset's properties in the stream.
1229
1230         --dest-config-path    string
1231                 Specifies a custom config path on the destination target.
1232                 The default is /var/lib/pve-zsync
1233     },
1234     list => qq{
1235 $PROGNAME list
1236
1237         Get a List of all scheduled Sync Jobs
1238     },
1239     status => qq{
1240 $PROGNAME status
1241
1242         Get the status of all scheduled Sync Jobs
1243     },
1244     help => qq{
1245 $PROGNAME help <cmd> [OPTIONS]
1246
1247     Get help about specified command.
1248
1249         <cmd>      string
1250                 Command name to get help about.
1251
1252         --verbose
1253                 Verbose output format.
1254     },
1255     enable => qq{
1256 $PROGNAME enable --source <string> [OPTIONS]
1257
1258     Enable a sync-job and reset all job-errors, if any.
1259
1260         --name      string
1261                 name of the sync job, if not set it is default
1262
1263         --source    string
1264                 the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1265     },
1266     disable => qq{
1267 $PROGNAME disable --source <string> [OPTIONS]
1268
1269     Disables (pauses) a sync-job
1270
1271         --name      string
1272                 name of the sync-job, if not set it is default
1273
1274         --source    string
1275                 the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1276     },
1277     printpod => "$PROGNAME printpod\n\n\tinternal command",
1278
1279 };
1280
1281 if (!$command) {
1282     usage(); die "\n";
1283 } elsif (!$cmd_help->{$command}) {
1284     print "ERROR: unknown command '$command'";
1285     usage(1); die "\n";
1286 }
1287
1288 my @arg = @ARGV;
1289 my $param = parse_argv(@arg);
1290
1291 sub check_params {
1292     for (@_) {
1293         die "$cmd_help->{$command}\n" if !$param->{$_};
1294     }
1295 }
1296
1297 if ($command eq 'destroy') {
1298     check_params(qw(source));
1299
1300     check_target($param->{source});
1301     destroy_job($param);
1302
1303 } elsif ($command eq 'sync') {
1304     check_params(qw(source dest));
1305
1306     check_target($param->{source});
1307     check_target($param->{dest});
1308     sync($param);
1309
1310 } elsif ($command eq 'create') {
1311     check_params(qw(source dest));
1312
1313     check_target($param->{source});
1314     check_target($param->{dest});
1315     init($param);
1316
1317 } elsif ($command eq 'status') {
1318     print status();
1319
1320 } elsif ($command eq 'list') {
1321     print list();
1322
1323 } elsif ($command eq 'help') {
1324     my $help_command = $ARGV[1];
1325
1326     if ($help_command && $cmd_help->{$help_command}) {
1327         die "$cmd_help->{$help_command}\n";
1328
1329     }
1330     if ($param->{verbose}) {
1331         exec("man $PROGNAME");
1332
1333     } else {
1334         usage(1);
1335
1336     }
1337
1338 } elsif ($command eq 'enable') {
1339     check_params(qw(source));
1340
1341     check_target($param->{source});
1342     enable_job($param);
1343
1344 } elsif ($command eq 'disable') {
1345     check_params(qw(source));
1346
1347     check_target($param->{source});
1348     disable_job($param);
1349
1350 } elsif ($command eq 'printpod') {
1351     print_pod();
1352 }
1353
1354 sub usage {
1355     my ($help) = @_;
1356
1357     print("ERROR:\tno command specified\n") if !$help;
1358     print("USAGE:\t$PROGNAME <COMMAND> [ARGS] [OPTIONS]\n");
1359     print("\t$PROGNAME help [<cmd>] [OPTIONS]\n\n");
1360     print("\t$PROGNAME create --dest <string> --source <string> [OPTIONS]\n");
1361     print("\t$PROGNAME destroy --source <string> [OPTIONS]\n");
1362     print("\t$PROGNAME disable --source <string> [OPTIONS]\n");
1363     print("\t$PROGNAME enable --source <string> [OPTIONS]\n");
1364     print("\t$PROGNAME list\n");
1365     print("\t$PROGNAME status\n");
1366     print("\t$PROGNAME sync --dest <string> --source <string> [OPTIONS]\n");
1367 }
1368
1369 sub check_target {
1370     my ($target) = @_;
1371     parse_target($target);
1372 }
1373
1374 sub print_pod {
1375
1376     my $synopsis = join("\n", sort values %$cmd_help);
1377     my $commands = join(", ", sort keys %$cmd_help);
1378
1379     print <<EOF;
1380 =head1 NAME
1381
1382 pve-zsync - PVE ZFS Storage Sync Tool
1383
1384 =head1 SYNOPSIS
1385
1386 pve-zsync <COMMAND> [ARGS] [OPTIONS]
1387
1388 Where <COMMAND> can be one of: $commands
1389
1390 =head1 DESCRIPTION
1391
1392 The pve-zsync tool can help you to sync your VMs or directories stored on ZFS
1393 between multiple servers.
1394
1395 pve-zsync is able to automatically configure CRON jobs, so that a periodic sync
1396 will be automatically triggered.
1397 The default sync interval is 15 min, if you want to change this value you can
1398 do this in F</etc/cron.d/pve-zsync>. If you need help to configure CRON tabs, see
1399 man crontab.
1400
1401 =head1 COMMANDS AND OPTIONS
1402
1403 $synopsis
1404
1405 =head1 EXAMPLES
1406
1407 Adds a job for syncing the local VM 100 to a remote server's ZFS pool named "tank":
1408     pve-zsync create --source=100 -dest=192.168.1.2:tank
1409
1410 =head1 IMPORTANT FILES
1411
1412 Cron jobs and config are stored in F</etc/cron.d/pve-zsync>
1413
1414 The VM configuration itself gets copied to the destination machines
1415 F</var/lib/pve-zsync/> path.
1416
1417 =head1 COPYRIGHT AND DISCLAIMER
1418
1419 Copyright (C) 2007-2021 Proxmox Server Solutions GmbH
1420
1421 This program is free software: you can redistribute it and/or modify it under
1422 the terms of the GNU Affero General Public License as published by the Free
1423 Software Foundation, either version 3 of the License, or (at your option) any
1424 later version.
1425
1426 This program is distributed in the hope that it will be useful, but WITHOUT ANY
1427 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
1428 PARTICULAR PURPOSE. See the GNU Affero General Public License for more
1429 details.
1430
1431 You should have received a copy of the GNU Affero General Public License along
1432 with this program. If not, see <http://www.gnu.org/licenses/>.
1433
1434 EOF
1435 }