pve-zsync

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl qw(:flock SEEK_END);
   7 use Getopt::Long qw(GetOptionsFromArray);
   8 use File::Path qw(make_path);
   9 use JSON;
  10 use IO::File;
  11 use String::ShellQuote 'shell_quote';
  12
  13 my $PROGNAME = "pve-zsync";
  14 my $CONFIG_PATH = "/var/lib/${PROGNAME}";
  15 my $STATE = "${CONFIG_PATH}/sync_state";
  16 my $CRONJOBS = "/etc/cron.d/$PROGNAME";
  17 my $PATH = "/usr/sbin";
  18 my $PVE_DIR = "/etc/pve/local";
  19 my $QEMU_CONF = "${PVE_DIR}/qemu-server";
  20 my $LXC_CONF = "${PVE_DIR}/lxc";
  21 my $PROG_PATH = "$PATH/${PROGNAME}";
  22 my $INTERVAL = 15;
  23 my $DEBUG;
  24
  25 BEGIN {
  26     $DEBUG = 0; # change default here. not above on declaration!
  27     $DEBUG ||= $ENV{ZSYNC_DEBUG};
  28     if ($DEBUG) {
  29         require Data::Dumper;
  30         Data::Dumper->import();
  31     }
  32 }
  33
  34 my $IPV4OCTET = "(?:25[0-5]|(?:[1-9]|1[0-9]|2[0-4])?[0-9])";
  35 my $IPV4RE = "(?:(?:$IPV4OCTET\\.){3}$IPV4OCTET)";
  36 my $IPV6H16 = "(?:[0-9a-fA-F]{1,4})";
  37 my $IPV6LS32 = "(?:(?:$IPV4RE|$IPV6H16:$IPV6H16))";
  38
  39 my $IPV6RE = "(?:" .
  40     "(?:(?:" .                             "(?:$IPV6H16:){6})$IPV6LS32)|" .
  41     "(?:(?:" .                           "::(?:$IPV6H16:){5})$IPV6LS32)|" .
  42     "(?:(?:(?:" .              "$IPV6H16)?::(?:$IPV6H16:){4})$IPV6LS32)|" .
  43     "(?:(?:(?:(?:$IPV6H16:){0,1}$IPV6H16)?::(?:$IPV6H16:){3})$IPV6LS32)|" .
  44     "(?:(?:(?:(?:$IPV6H16:){0,2}$IPV6H16)?::(?:$IPV6H16:){2})$IPV6LS32)|" .
  45     "(?:(?:(?:(?:$IPV6H16:){0,3}$IPV6H16)?::(?:$IPV6H16:){1})$IPV6LS32)|" .
  46     "(?:(?:(?:(?:$IPV6H16:){0,4}$IPV6H16)?::" .           ")$IPV6LS32)|" .
  47     "(?:(?:(?:(?:$IPV6H16:){0,5}$IPV6H16)?::" .            ")$IPV6H16)|" .
  48     "(?:(?:(?:(?:$IPV6H16:){0,6}$IPV6H16)?::" .                    ")))";
  49
  50 my $HOSTv4RE0 = "(?:[\\w\\.\\-_]+|$IPV4RE)";       # hostname or ipv4 address
  51 my $HOSTv4RE1 = "(?:$HOSTv4RE0|\\[$HOSTv4RE0\\])"; # these may be in brackets, too
  52 my $HOSTRE = "(?:$HOSTv4RE1|\\[$IPV6RE\\])";       # ipv6 must always be in brackets
  53 # targets are either a VMID, or a 'host:zpool/path' with 'host:' being optional
  54 my $TARGETRE = qr!^(?:($HOSTRE):)?(\d+|(?:[\w\-_]+)(/.+)?)$!;
  55
  56 my $DISK_KEY_RE = qr/^(?:(?:(?:virtio|ide|scsi|sata|efidisk|mp)\d+)|rootfs): /;
  57
  58 my $INSTANCE_ID = get_instance_id($$);
  59
  60 my $command = $ARGV[0];
  61
  62 if (defined($command) && $command ne 'help' && $command ne 'printpod') {
  63     check_bin ('cstream');
  64     check_bin ('zfs');
  65     check_bin ('ssh');
  66     check_bin ('scp');
  67 }
  68
  69 $SIG{TERM} = $SIG{QUIT} = $SIG{PIPE} = $SIG{HUP} = $SIG{KILL} = $SIG{INT} =  sub {
  70     die "Signaled, aborting sync: $!\n";
  71 };
  72
  73 sub check_bin {
  74     my ($bin)  = @_;
  75
  76     foreach my $p (split (/:/, $ENV{PATH})) {
  77         my $fn = "$p/$bin";
  78         if (-x $fn) {
  79             return $fn;
  80         }
  81     }
  82
  83     die "unable to find command '$bin'\n";
  84 }
  85
  86 sub read_file {
  87     my ($filename, $one_line_only) = @_;
  88
  89     my $fh = IO::File->new($filename, "r")
  90         or die "Could not open file ${filename}: $!\n";
  91
  92     my $text = $one_line_only ? <$fh> : [ <$fh> ];
  93
  94     close($fh);
  95
  96     return $text;
  97 }
  98
  99 sub cut_target_width {
 100     my ($path, $maxlen) = @_;
 101     $path =~ s@/+@/@g;
 102
 103     return $path if length($path) <= $maxlen;
 104
 105     return '..'.substr($path, -$maxlen+2) if $path !~ m@/@;
 106
 107     $path =~ s@/([^/]+/?)$@@;
 108     my $tail = $1;
 109
 110     if (length($tail)+3 == $maxlen) {
 111         return "../$tail";
 112     } elsif (length($tail)+2 >= $maxlen) {
 113         return '..'.substr($tail, -$maxlen+2)
 114     }
 115
 116     $path =~ s@(/[^/]+)(?:/|$)@@;
 117     my $head = $1;
 118     my $both = length($head) + length($tail);
 119     my $remaining = $maxlen-$both-4; # -4 for "/../"
 120
 121     if ($remaining < 0) {
 122         return substr($head, 0, $maxlen - length($tail) - 3) . "../$tail"; # -3 for "../"
 123     }
 124
 125     substr($path, ($remaining/2), (length($path)-$remaining), '..');
 126     return "$head/" . $path . "/$tail";
 127 }
 128
 129 sub locked {
 130     my ($lock_fn, $code) = @_;
 131
 132     my $lock_fh = IO::File->new("> $lock_fn");
 133
 134     flock($lock_fh, LOCK_EX) || die "Couldn't acquire lock - $!\n";
 135     my $res = eval { $code->() };
 136     my $err = $@;
 137
 138     flock($lock_fh, LOCK_UN) || warn "Error unlocking - $!\n";
 139     die "$err" if $err;
 140
 141     close($lock_fh);
 142     return $res;
 143 }
 144
 145 sub get_status {
 146     my ($source, $name, $status) = @_;
 147
 148     if ($status->{$source->{all}}->{$name}->{status}) {
 149         return $status;
 150     }
 151
 152     return undef;
 153 }
 154
 155 sub check_dataset_exists {
 156     my ($dataset, $ip, $user) = @_;
 157
 158     my $cmd = [];
 159
 160     if ($ip) {
 161         push @$cmd, 'ssh', "$user\@$ip", '--';
 162     }
 163     push @$cmd, 'zfs', 'list', '-H', '--', $dataset;
 164     eval {
 165         run_cmd($cmd);
 166     };
 167
 168     if ($@) {
 169         return 0;
 170     }
 171     return 1;
 172 }
 173
 174 sub create_file_system {
 175     my ($file_system, $ip, $user) = @_;
 176
 177     my $cmd = [];
 178
 179     if ($ip) {
 180         push @$cmd, 'ssh', "$user\@$ip", '--';
 181     }
 182     push @$cmd, 'zfs', 'create', $file_system;
 183
 184     run_cmd($cmd);
 185 }
 186
 187 sub parse_target {
 188     my ($text) = @_;
 189
 190     my $errstr = "$text : is not a valid input! Use [IP:]<VMID> or [IP:]<ZFSPool>[/Path]";
 191     my $target = {};
 192
 193     if ($text !~ $TARGETRE) {
 194         die "$errstr\n";
 195     }
 196     $target->{all} = $2;
 197     $target->{ip} = $1 if $1;
 198     my @parts = split('/', $2);
 199
 200     $target->{ip} =~ s/^\[(.*)\]$/$1/ if $target->{ip};
 201
 202     my $pool = $target->{pool} = shift(@parts);
 203     die "$errstr\n" if !$pool;
 204
 205     if ($pool =~ m/^\d+$/) {
 206         $target->{vmid} = $pool;
 207         delete $target->{pool};
 208     }
 209
 210     return $target if (@parts == 0);
 211     $target->{last_part} = pop(@parts);
 212
 213     if ($target->{ip}) {
 214         pop(@parts);
 215     }
 216     if (@parts > 0) {
 217         $target->{path} = join('/', @parts);
 218     }
 219
 220     return $target;
 221 }
 222
 223 sub read_cron {
 224
 225     #This is for the first use to init file;
 226     if (!-e $CRONJOBS) {
 227         my $new_fh = IO::File->new("> $CRONJOBS");
 228         die "Could not create $CRONJOBS: $!\n" if !$new_fh;
 229         close($new_fh);
 230         return undef;
 231     }
 232
 233     my $text = read_file($CRONJOBS, 0);
 234
 235     return encode_cron(@{$text});
 236 }
 237
 238 sub parse_argv {
 239     my (@arg) = @_;
 240
 241     my $param = {
 242         dest => undef,
 243         source => undef,
 244         verbose => undef,
 245         limit => undef,
 246         maxsnap => undef,
 247         name => undef,
 248         skip => undef,
 249         method => undef,
 250         source_user => undef,
 251         dest_user => undef,
 252         prepend_storage_id => undef,
 253         properties => undef,
 254         dest_config_path => undef,
 255     };
 256
 257     my ($ret) = GetOptionsFromArray(
 258         \@arg,
 259         'dest=s' => \$param->{dest},
 260         'source=s' => \$param->{source},
 261         'verbose' => \$param->{verbose},
 262         'limit=i' => \$param->{limit},
 263         'maxsnap=i' => \$param->{maxsnap},
 264         'name=s' => \$param->{name},
 265         'skip' => \$param->{skip},
 266         'method=s' => \$param->{method},
 267         'source-user=s' => \$param->{source_user},
 268         'dest-user=s' => \$param->{dest_user},
 269         'prepend-storage-id' => \$param->{prepend_storage_id},
 270         'properties' => \$param->{properties},
 271         'dest-config-path=s' => \$param->{dest_config_path},
 272     );
 273
 274     die "can't parse options\n" if $ret == 0;
 275
 276     $param->{name} //= "default";
 277     $param->{maxsnap} //= 1;
 278     $param->{method} //= "ssh";
 279     $param->{source_user} //= "root";
 280     $param->{dest_user} //= "root";
 281
 282     return $param;
 283 }
 284
 285 sub add_state_to_job {
 286     my ($job) = @_;
 287
 288     my $states = read_state();
 289     my $state = $states->{$job->{source}}->{$job->{name}};
 290
 291     $job->{state} = $state->{state};
 292     $job->{lsync} = $state->{lsync};
 293     $job->{vm_type} = $state->{vm_type};
 294     $job->{instance_id} = $state->{instance_id};
 295
 296     for (my $i = 0; $state->{"snap$i"}; $i++) {
 297         $job->{"snap$i"} = $state->{"snap$i"};
 298     }
 299
 300     return $job;
 301 }
 302
 303 sub encode_cron {
 304     my (@text) = @_;
 305
 306     my $cfg = {};
 307
 308     while (my $line = shift(@text)) {
 309
 310         my @arg = split('\s', $line);
 311         my $param = parse_argv(@arg);
 312
 313         if ($param->{source} && $param->{dest}) {
 314             my $source = delete $param->{source};
 315             my $name = delete $param->{name};
 316
 317             $cfg->{$source}->{$name} = $param;
 318         }
 319     }
 320
 321     return $cfg;
 322 }
 323
 324 sub param_to_job {
 325     my ($param) = @_;
 326
 327     my $job = {};
 328
 329     my $source = parse_target($param->{source});
 330     my $dest;
 331     $dest = parse_target($param->{dest}) if $param->{dest};
 332
 333     $job->{name} = !$param->{name} ? "default" : $param->{name};
 334     $job->{dest} = $param->{dest} if $param->{dest};
 335     $job->{method} = "local" if !$dest->{ip} && !$source->{ip};
 336     $job->{method} = "ssh" if !$job->{method};
 337     $job->{limit} = $param->{limit};
 338     $job->{maxsnap} = $param->{maxsnap};
 339     $job->{source} = $param->{source};
 340     $job->{source_user} = $param->{source_user};
 341     $job->{dest_user} = $param->{dest_user};
 342     $job->{prepend_storage_id} = !!$param->{prepend_storage_id};
 343     $job->{properties} = !!$param->{properties};
 344     $job->{dest_config_path} = $param->{dest_config_path} if $param->{dest_config_path};
 345
 346     return $job;
 347 }
 348
 349 sub read_state {
 350
 351     if (!-e $STATE) {
 352         make_path $CONFIG_PATH;
 353         my $new_fh = IO::File->new("> $STATE");
 354         die "Could not create $STATE: $!\n" if !$new_fh;
 355         print $new_fh "{}";
 356         close($new_fh);
 357         return undef;
 358     }
 359
 360     my $text = read_file($STATE, 1);
 361     return decode_json($text);
 362 }
 363
 364 sub update_state {
 365     my ($job) = @_;
 366
 367     my $text = eval { read_file($STATE, 1); };
 368
 369     my $out_fh = IO::File->new("> $STATE.new");
 370     die "Could not open file ${STATE}.new: $!\n" if !$out_fh;
 371
 372     my $states = {};
 373     my $state = {};
 374     if ($text){
 375         $states = decode_json($text);
 376         $state = $states->{$job->{source}}->{$job->{name}};
 377     }
 378
 379     if ($job->{state} ne "del") {
 380         $state->{state} = $job->{state};
 381         $state->{lsync} = $job->{lsync};
 382         $state->{instance_id} = $job->{instance_id};
 383         $state->{vm_type} = $job->{vm_type};
 384
 385         for (my $i = 0; $job->{"snap$i"} ; $i++) {
 386             $state->{"snap$i"} = $job->{"snap$i"};
 387         }
 388         $states->{$job->{source}}->{$job->{name}} = $state;
 389     } else {
 390
 391         delete $states->{$job->{source}}->{$job->{name}};
 392         delete $states->{$job->{source}} if !keys %{$states->{$job->{source}}};
 393     }
 394
 395     $text = encode_json($states);
 396     print $out_fh $text;
 397
 398     close($out_fh);
 399     rename "$STATE.new", $STATE;
 400
 401     return $states;
 402 }
 403
 404 sub update_cron {
 405     my ($job) = @_;
 406
 407     my $updated;
 408     my $has_header;
 409     my $line_no = 0;
 410     my $text = "";
 411     my $header = "SHELL=/bin/sh\n";
 412     $header .= "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n\n";
 413
 414     my $current = read_file($CRONJOBS, 0);
 415
 416     foreach my $line (@{$current}) {
 417         chomp($line);
 418         if ($line =~ m/source $job->{source} .*name $job->{name} /) {
 419             $updated = 1;
 420             next if $job->{state} eq "del";
 421             $text .= format_job($job, $line);
 422         } else {
 423             if (($line_no < 3) && ($line =~ /^(PATH|SHELL)/ )) {
 424                 $has_header = 1;
 425             }
 426             $text .= "$line\n";
 427         }
 428         $line_no++;
 429     }
 430
 431     if (!$has_header) {
 432         $text = "$header$text";
 433     }
 434
 435     if (!$updated) {
 436         $text .= format_job($job);
 437     }
 438     my $new_fh = IO::File->new("> ${CRONJOBS}.new");
 439     die "Could not open file ${CRONJOBS}.new: $!\n" if !$new_fh;
 440
 441     print $new_fh $text or die "can't write to $CRONJOBS.new: $!\n";
 442     close ($new_fh);
 443
 444     rename "${CRONJOBS}.new", $CRONJOBS or die "can't move $CRONJOBS.new: $!\n";
 445 }
 446
 447 sub format_job {
 448     my ($job, $line) = @_;
 449     my $text = "";
 450
 451     if ($job->{state} eq "stopped") {
 452         $text = "#";
 453     }
 454     if ($line) {
 455         $line =~ /^#*\s*((?:\S+\s+){4}\S+)\s+root/;
 456         $text .= $1;
 457     } else {
 458         $text .= "*/$INTERVAL * * * *";
 459     }
 460     $text .= " root";
 461     $text .= " $PROGNAME sync --source $job->{source} --dest $job->{dest}";
 462     $text .= " --name $job->{name} --maxsnap $job->{maxsnap}";
 463     $text .= " --limit $job->{limit}" if $job->{limit};
 464     $text .= " --method $job->{method}";
 465     $text .= " --verbose" if $job->{verbose};
 466     $text .= " --source-user $job->{source_user}";
 467     $text .= " --dest-user $job->{dest_user}";
 468     $text .= " --prepend-storage-id" if $job->{prepend_storage_id};
 469     $text .= " --properties" if $job->{properties};
 470     $text .= " --dest-config-path $job->{dest_config_path}" if $job->{dest_config_path};
 471     $text .= "\n";
 472
 473     return $text;
 474 }
 475
 476 sub list {
 477
 478     my $cfg = read_cron();
 479
 480     my $list = sprintf("%-25s%-25s%-10s%-20s%-6s%-5s\n" , "SOURCE", "NAME", "STATE", "LAST SYNC", "TYPE", "CON");
 481
 482     my $states = read_state();
 483     foreach my $source (sort keys%{$cfg}) {
 484         foreach my $name (sort keys%{$cfg->{$source}}) {
 485             $list .= sprintf("%-25s", cut_target_width($source, 25));
 486             $list .= sprintf("%-25s", cut_target_width($name, 25));
 487             $list .= sprintf("%-10s", $states->{$source}->{$name}->{state});
 488             $list .= sprintf("%-20s", $states->{$source}->{$name}->{lsync});
 489             $list .= sprintf("%-6s", defined($states->{$source}->{$name}->{vm_type}) ? $states->{$source}->{$name}->{vm_type} : "undef");
 490             $list .= sprintf("%-5s\n", $cfg->{$source}->{$name}->{method});
 491         }
 492     }
 493
 494     return $list;
 495 }
 496
 497 sub vm_exists {
 498     my ($target, $user) = @_;
 499
 500     return undef if !defined($target->{vmid});
 501
 502     my $conf_fn = "$target->{vmid}.conf";
 503
 504     if ($target->{ip}) {
 505         my @cmd = ('ssh', "$user\@$target->{ip}", '--', '/bin/ls');
 506         return "qemu" if eval { run_cmd([@cmd, "$QEMU_CONF/$conf_fn"]) };
 507         return "lxc" if  eval { run_cmd([@cmd, "$LXC_CONF/$conf_fn"]) };
 508     } else {
 509         return "qemu" if -f "$QEMU_CONF/$conf_fn";
 510         return "lxc" if -f "$LXC_CONF/$conf_fn";
 511     }
 512
 513     return undef;
 514 }
 515
 516 sub init {
 517     my ($param) = @_;
 518
 519     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 520         my $cfg = read_cron();
 521
 522         my $job = param_to_job($param);
 523
 524         $job->{state} = "ok";
 525         $job->{lsync} = 0;
 526
 527         my $source = parse_target($param->{source});
 528         my $dest = parse_target($param->{dest});
 529
 530         if (my $ip =  $dest->{ip}) {
 531             run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{dest_user}\@$ip"]);
 532         }
 533
 534         if (my $ip =  $source->{ip}) {
 535             run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{source_user}\@$ip"]);
 536         }
 537
 538         die "Pool $dest->{all} does not exist\n"
 539             if !check_dataset_exists($dest->{all}, $dest->{ip}, $param->{dest_user});
 540
 541         if (!defined($source->{vmid})) {
 542             die "Pool $source->{all} does not exist\n"
 543                 if !check_dataset_exists($source->{all}, $source->{ip}, $param->{source_user});
 544         }
 545
 546         my $vm_type = vm_exists($source, $param->{source_user});
 547         $job->{vm_type} = $vm_type;
 548         $source->{vm_type} = $vm_type;
 549
 550         die "VM $source->{vmid} doesn't exist\n" if $source->{vmid} && !$vm_type;
 551
 552         die "Config already exists\n" if $cfg->{$job->{source}}->{$job->{name}};
 553
 554         #check if vm has zfs disks if not die;
 555         get_disks($source, $param->{source_user}) if $source->{vmid};
 556
 557         update_cron($job);
 558         update_state($job);
 559     }); #cron and state lock
 560
 561     return if $param->{skip};
 562
 563     eval { sync($param) };
 564     if (my $err = $@) {
 565         destroy_job($param);
 566         print $err;
 567     }
 568 }
 569
 570 sub get_job {
 571     my ($param) = @_;
 572
 573     my $cfg = read_cron();
 574
 575     if (!$cfg->{$param->{source}}->{$param->{name}}) {
 576         die "Job  with source $param->{source} and name $param->{name} does not exist\n" ;
 577     }
 578     my $job = $cfg->{$param->{source}}->{$param->{name}};
 579     $job->{name} = $param->{name};
 580     $job->{source} = $param->{source};
 581     $job = add_state_to_job($job);
 582
 583     return $job;
 584 }
 585
 586 sub destroy_job {
 587     my ($param) = @_;
 588
 589     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 590         my $job = get_job($param);
 591         $job->{state} = "del";
 592
 593         update_cron($job);
 594         update_state($job);
 595     });
 596 }
 597
 598 sub get_instance_id {
 599     my ($pid) = @_;
 600
 601     my $stat = read_file("/proc/$pid/stat", 1)
 602         or die "unable to read process stats\n";
 603     my $boot_id = read_file("/proc/sys/kernel/random/boot_id", 1)
 604         or die "unable to read boot ID\n";
 605
 606     my $stats = [ split(/\s+/, $stat) ];
 607     my $starttime = $stats->[21];
 608     chomp($boot_id);
 609
 610     return "${pid}:${starttime}:${boot_id}";
 611 }
 612
 613 sub instance_exists {
 614     my ($instance_id) = @_;
 615
 616     if (defined($instance_id) && $instance_id =~ m/^([1-9][0-9]*):/) {
 617         my $pid = $1;
 618         my $actual_id = eval { get_instance_id($pid); };
 619         return defined($actual_id) && $actual_id eq $instance_id;
 620     }
 621
 622     return 0;
 623 }
 624
 625 sub sync {
 626     my ($param) = @_;
 627
 628     my $job;
 629
 630     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 631         eval { $job = get_job($param) };
 632
 633         if ($job) {
 634             my $state = $job->{state} // 'ok';
 635             $state = 'ok' if !instance_exists($job->{instance_id});
 636
 637             if ($state eq "syncing" || $state eq "waiting") {
 638                 die "Job --source $param->{source} --name $param->{name} is already scheduled to sync\n";
 639             }
 640
 641             $job->{state} = "waiting";
 642             $job->{instance_id} = $INSTANCE_ID;
 643
 644             update_state($job);
 645         }
 646     });
 647
 648     locked("$CONFIG_PATH/sync.lock", sub {
 649
 650         my $date = get_date();
 651
 652         my $dest;
 653         my $source;
 654         my $vm_type;
 655
 656         locked("$CONFIG_PATH/cron_and_state.lock", sub {
 657             #job might've changed while we waited for the sync lock, but we can be sure it's not syncing
 658             eval { $job = get_job($param); };
 659
 660             if ($job && defined($job->{state}) && $job->{state} eq "stopped") {
 661                 die "Job --source $param->{source} --name $param->{name} has been disabled\n";
 662             }
 663
 664             $dest = parse_target($param->{dest});
 665             $source = parse_target($param->{source});
 666
 667             $vm_type = vm_exists($source, $param->{source_user});
 668             $source->{vm_type} = $vm_type;
 669
 670             if ($job) {
 671                 $job->{state} = "syncing";
 672                 $job->{vm_type} = $vm_type if !$job->{vm_type};
 673                 update_state($job);
 674             }
 675         }); #cron and state lock
 676
 677         my $sync_path = sub {
 678             my ($source, $dest, $job, $param, $date) = @_;
 679
 680             my $dest_dataset = target_dataset($source, $dest);
 681
 682             ($dest->{old_snap}, $dest->{last_snap}) = snapshot_get(
 683                 $dest_dataset,
 684                 $param->{maxsnap},
 685                 $param->{name},
 686                 $dest->{ip},
 687                 $param->{dest_user},
 688             );
 689
 690             prepare_prepended_target($source, $dest, $param->{dest_user}) if defined($dest->{prepend});
 691
 692             snapshot_add($source, $dest, $param->{name}, $date, $param->{source_user}, $param->{dest_user});
 693
 694             send_image($source, $dest, $param);
 695
 696             for my $old_snap (@{$dest->{old_snap}}) {
 697                 snapshot_destroy($source->{all}, $old_snap, $source->{ip}, $param->{source_user});
 698                 snapshot_destroy($dest_dataset, $old_snap, $dest->{ip}, $param->{dest_user});
 699             }
 700         };
 701
 702         eval{
 703             if ($source->{vmid}) {
 704                 die "VM $source->{vmid} doesn't exist\n" if !$vm_type;
 705                 die "source-user has to be root for syncing VMs\n" if ($param->{source_user} ne "root");
 706                 my $disks = get_disks($source, $param->{source_user});
 707
 708                 foreach my $disk (sort keys %{$disks}) {
 709                     $source->{all} = $disks->{$disk}->{all};
 710                     $source->{pool} = $disks->{$disk}->{pool};
 711                     $source->{path} = $disks->{$disk}->{path} if $disks->{$disk}->{path};
 712                     $source->{last_part} = $disks->{$disk}->{last_part};
 713
 714                     $dest->{prepend} = $disks->{$disk}->{storage_id}
 715                         if $param->{prepend_storage_id};
 716
 717                     &$sync_path($source, $dest, $job, $param, $date);
 718                 }
 719                 if ($param->{method} eq "ssh" && ($source->{ip} || $dest->{ip})) {
 720                     send_config($source, $dest,'ssh', $param->{source_user}, $param->{dest_user}, $param->{dest_config_path});
 721                 } else {
 722                     send_config($source, $dest,'local', $param->{source_user}, $param->{dest_user}, $param->{dest_config_path});
 723                 }
 724             } else {
 725                 &$sync_path($source, $dest, $job, $param, $date);
 726             }
 727         };
 728         if (my $err = $@) {
 729             locked("$CONFIG_PATH/cron_and_state.lock", sub {
 730                 eval { $job = get_job($param); };
 731                 if ($job) {
 732                     $job->{state} = "error";
 733                     delete $job->{instance_id};
 734                     update_state($job);
 735                 }
 736             });
 737             print "Job --source $param->{source} --name $param->{name} got an ERROR!!!\nERROR Message:\n";
 738             die "$err\n";
 739         }
 740
 741         locked("$CONFIG_PATH/cron_and_state.lock", sub {
 742             eval { $job = get_job($param); };
 743             if ($job) {
 744                 if (defined($job->{state}) && $job->{state} eq "stopped") {
 745                     $job->{state} = "stopped";
 746                 } else {
 747                     $job->{state} = "ok";
 748                 }
 749                 $job->{lsync} = $date;
 750                 delete $job->{instance_id};
 751                 update_state($job);
 752             }
 753         });
 754     }); #sync lock
 755 }
 756
 757 sub snapshot_get{
 758     my ($dataset, $max_snap, $name, $ip, $user) = @_;
 759
 760     my $cmd = [];
 761     push @$cmd, 'ssh', "$user\@$ip", '--', if $ip;
 762     push @$cmd, 'zfs', 'list', '-r', '-t', 'snapshot', '-Ho', 'name', '-S', 'creation';
 763     push @$cmd, $dataset;
 764
 765     my $raw;
 766     eval {$raw = run_cmd($cmd)};
 767     if (my $erro =$@) { #this means the volume doesn't exist on dest yet
 768        return undef;
 769     }
 770
 771     my $index = 0;
 772     my $line = "";
 773     my $last_snap = undef;
 774     my $old_snap = [];
 775
 776     while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
 777         $line = $1;
 778         if ($line =~ m/@(.*)$/) {
 779             $last_snap = $1 if (!$last_snap);
 780         }
 781         if ($line =~ m/(rep_\Q${name}\E_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})$/) {
 782             # interpreted as infinity
 783             last if $max_snap <= 0;
 784
 785             my $snap = $1;
 786             $index++;
 787
 788             if ($index >= $max_snap) {
 789                 push @{$old_snap}, $snap;
 790             }
 791         }
 792     }
 793
 794     return ($old_snap, $last_snap) if $last_snap;
 795
 796     return undef;
 797 }
 798
 799 sub snapshot_add {
 800     my ($source, $dest, $name, $date, $source_user, $dest_user) = @_;
 801
 802     my $snap_name = "rep_$name\_".$date;
 803
 804     $source->{new_snap} = $snap_name;
 805
 806     my $path = "$source->{all}\@$snap_name";
 807
 808     my $cmd = [];
 809     push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--', if $source->{ip};
 810     push @$cmd, 'zfs', 'snapshot', $path;
 811     eval{
 812         run_cmd($cmd);
 813     };
 814
 815     if (my $err = $@) {
 816         snapshot_destroy($source->{all}, $snap_name, $source->{ip}, $source_user);
 817         die "$err\n";
 818     }
 819 }
 820
 821 sub get_disks {
 822     my ($target, $user) = @_;
 823
 824     my $cmd = [];
 825     push @$cmd, 'ssh', "$user\@$target->{ip}", '--', if $target->{ip};
 826
 827     if ($target->{vm_type} eq 'qemu') {
 828         push @$cmd, 'qm', 'config', $target->{vmid};
 829     } elsif ($target->{vm_type} eq 'lxc') {
 830         push @$cmd, 'pct', 'config', $target->{vmid};
 831     } else {
 832         die "VM Type unknown\n";
 833     }
 834
 835     my $res = run_cmd($cmd);
 836
 837     my $disks = parse_disks($res, $target->{ip}, $target->{vm_type}, $user);
 838
 839     return $disks;
 840 }
 841
 842 sub run_cmd {
 843     my ($cmd) = @_;
 844     print "Start CMD\n" if $DEBUG;
 845     print Dumper $cmd if $DEBUG;
 846     if (ref($cmd) eq 'ARRAY') {
 847         $cmd = join(' ', map { ref($_) ? $$_ : shell_quote($_) } @$cmd);
 848     }
 849     my $output = `$cmd 2>&1`;
 850
 851     die "COMMAND:\n\t$cmd\nGET ERROR:\n\t$output" if 0 != $?;
 852
 853     chomp($output);
 854     print Dumper $output if $DEBUG;
 855     print "END CMD\n" if $DEBUG;
 856     return $output;
 857 }
 858
 859 sub parse_disks {
 860     my ($text, $ip, $vm_type, $user) = @_;
 861
 862     my $disks;
 863
 864     my $num = 0;
 865     while ($text && $text =~ s/^(.*?)(\n|$)//) {
 866         my $line = $1;
 867
 868         next if $line =~ /media=cdrom/;
 869         next if $line !~ m/$DISK_KEY_RE/;
 870
 871         #QEMU if backup is not set include in  sync
 872         next if $vm_type eq 'qemu' && ($line =~ m/backup=(?i:0|no|off|false)/);
 873
 874         #LXC if backup is not set do no in sync
 875         next if $vm_type eq 'lxc' && ($line =~ m/^mp\d:/) && ($line !~ m/backup=(?i:1|yes|on|true)/);
 876
 877         my $disk = undef;
 878         my $stor = undef;
 879         if($line =~ m/$DISK_KEY_RE(.*)$/) {
 880             my @parameter = split(/,/,$1);
 881
 882             foreach my $opt (@parameter) {
 883                 if ($opt =~ m/^(?:file=|volume=)?([^:]+):([A-Za-z0-9\-]+)$/){
 884                     $disk = $2;
 885                     $stor = $1;
 886                     last;
 887                 }
 888             }
 889         }
 890         if (!defined($disk) || !defined($stor)) {
 891             print "Disk: \"$line\" has no valid zfs dataset format and will be skipped\n";
 892             next;
 893         }
 894
 895         my $cmd = [];
 896         push @$cmd, 'ssh', "$user\@$ip", '--' if $ip;
 897         push @$cmd, 'pvesm', 'path', "$stor:$disk";
 898         my $path = run_cmd($cmd);
 899
 900         die "Get no path from pvesm path $stor:$disk\n" if !$path;
 901
 902         $disks->{$num}->{storage_id} = $stor;
 903
 904         if ($vm_type eq 'qemu' && $path =~ m/^\/dev\/zvol\/(\w+.*)(\/$disk)$/) {
 905
 906             my @array = split('/', $1);
 907             $disks->{$num}->{pool} = shift(@array);
 908             $disks->{$num}->{all} = $disks->{$num}->{pool};
 909             if (0 < @array) {
 910                 $disks->{$num}->{path} = join('/', @array);
 911                 $disks->{$num}->{all} .= "\/$disks->{$num}->{path}";
 912             }
 913             $disks->{$num}->{last_part} = $disk;
 914             $disks->{$num}->{all} .= "\/$disk";
 915
 916             $num++;
 917         } elsif ($vm_type eq 'lxc' && $path =~ m/^\/(\w+.+)(\/(\w+.*))*(\/$disk)$/) {
 918
 919             $disks->{$num}->{pool} = $1;
 920             $disks->{$num}->{all} = $disks->{$num}->{pool};
 921
 922             if ($2) {
 923                 $disks->{$num}->{path} = $3;
 924                 $disks->{$num}->{all} .= "\/$disks->{$num}->{path}";
 925             }
 926
 927             $disks->{$num}->{last_part} = $disk;
 928             $disks->{$num}->{all} .= "\/$disk";
 929
 930             $num++;
 931
 932         } else {
 933             die "ERROR: in path\n";
 934         }
 935     }
 936
 937     die "Vm include no disk on zfs.\n" if !$disks->{0};
 938     return $disks;
 939 }
 940
 941 # how the corresponding dataset is named on the target
 942 sub target_dataset {
 943     my ($source, $dest) = @_;
 944
 945     my $target = "$dest->{all}";
 946     $target .= "/$dest->{prepend}" if defined($dest->{prepend});
 947     $target .= "/$source->{last_part}" if $source->{last_part};
 948     $target =~ s!/+!/!g;
 949
 950     return $target;
 951 }
 952
 953 # create the parent dataset for the actual target
 954 sub prepare_prepended_target {
 955     my ($source, $dest, $dest_user) = @_;
 956
 957     die "internal error - not a prepended target\n" if !defined($dest->{prepend});
 958
 959     # The parent dataset shouldn't be the actual target.
 960     die "internal error - no last_part for source\n" if !$source->{last_part};
 961
 962     my $target = "$dest->{all}/$dest->{prepend}";
 963     $target =~ s!/+!/!g;
 964
 965     return if check_dataset_exists($target, $dest->{ip}, $dest_user);
 966
 967     create_file_system($target, $dest->{ip}, $dest_user);
 968 }
 969
 970 sub snapshot_destroy {
 971     my ($dataset, $snap, $ip, $user) = @_;
 972
 973     my @zfscmd = ('zfs', 'destroy');
 974     my $snapshot = "$dataset\@$snap";
 975
 976     eval {
 977         if ($ip) {
 978             run_cmd(['ssh', "$user\@$ip", '--', @zfscmd, $snapshot]);
 979         } else {
 980             run_cmd([@zfscmd, $snapshot]);
 981         }
 982     };
 983     if (my $erro = $@) {
 984         warn "WARN: $erro";
 985     }
 986 }
 987
 988 # check if snapshot for incremental sync exist on source side
 989 sub snapshot_exist {
 990     my ($source , $dest, $method, $source_user) = @_;
 991
 992     my $cmd = [];
 993     push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--' if $source->{ip};
 994     push @$cmd, 'zfs', 'list', '-rt', 'snapshot', '-Ho', 'name';
 995
 996     my $path = $source->{all};
 997     $path .= "\@$dest->{last_snap}";
 998
 999     push @$cmd, $path;
1000
1001     eval {run_cmd($cmd)};
1002     if (my $erro =$@) {
1003         warn "WARN: $erro";
1004         return undef;
1005     }
1006     return 1;
1007 }
1008
1009 sub send_image {
1010     my ($source, $dest, $param) = @_;
1011
1012     my $cmd = [];
1013
1014     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{source_user}\@$source->{ip}", '--' if $source->{ip};
1015     push @$cmd, 'zfs', 'send';
1016     push @$cmd, '-p', if $param->{properties};
1017     push @$cmd, '-v' if $param->{verbose};
1018
1019     if($dest->{last_snap} && snapshot_exist($source , $dest, $param->{method}, $param->{source_user})) {
1020         push @$cmd, '-i', "$source->{all}\@$dest->{last_snap}";
1021     }
1022     push @$cmd, '--', "$source->{all}\@$source->{new_snap}";
1023
1024     if ($param->{limit}){
1025         my $bwl = $param->{limit}*1024;
1026         push @$cmd, \'|', 'cstream', '-t', $bwl;
1027     }
1028     my $target = target_dataset($source, $dest);
1029
1030     push @$cmd, \'|';
1031     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{dest_user}\@$dest->{ip}", '--' if $dest->{ip};
1032     push @$cmd, 'zfs', 'recv', '-F', '--';
1033     push @$cmd, "$target";
1034
1035     eval {
1036         run_cmd($cmd)
1037     };
1038
1039     if (my $erro = $@) {
1040         snapshot_destroy($source->{all}, $source->{new_snap}, $source->{ip}, $param->{source_user});
1041         die $erro;
1042     };
1043 }
1044
1045
1046 sub send_config{
1047     my ($source, $dest, $method, $source_user, $dest_user, $dest_config_path) = @_;
1048
1049     my $source_target = $source->{vm_type} eq 'qemu' ? "$QEMU_CONF/$source->{vmid}.conf": "$LXC_CONF/$source->{vmid}.conf";
1050     my $dest_target_new ="$source->{vmid}.conf.$source->{vm_type}.$source->{new_snap}";
1051
1052     my $config_dir = $dest_config_path // $CONFIG_PATH;
1053     $config_dir .= "/$dest->{last_part}" if $dest->{last_part};
1054
1055     $dest_target_new = $config_dir.'/'.$dest_target_new;
1056
1057     if ($method eq 'ssh'){
1058         if ($dest->{ip} && $source->{ip}) {
1059             run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'mkdir', '-p', '--', $config_dir]);
1060             run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", "$dest_user\@[$dest->{ip}]:$dest_target_new"]);
1061         } elsif ($dest->{ip}) {
1062             run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'mkdir', '-p', '--', $config_dir]);
1063             run_cmd(['scp', '--', $source_target, "$dest_user\@[$dest->{ip}]:$dest_target_new"]);
1064         } elsif ($source->{ip}) {
1065             run_cmd(['mkdir', '-p', '--', $config_dir]);
1066             run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", $dest_target_new]);
1067         }
1068
1069         for my $old_snap (@{$dest->{old_snap}}) {
1070             my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.${old_snap}";
1071             if($dest->{ip}){
1072                 run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'rm', '-f', '--', $dest_target_old]);
1073             } else {
1074                 run_cmd(['rm', '-f', '--', $dest_target_old]);
1075             }
1076         }
1077     } elsif ($method eq 'local') {
1078         run_cmd(['mkdir', '-p', '--', $config_dir]);
1079         run_cmd(['cp', $source_target, $dest_target_new]);
1080     }
1081 }
1082
1083 sub get_date {
1084     my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
1085     my $datestamp = sprintf ("%04d-%02d-%02d_%02d:%02d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
1086
1087     return $datestamp;
1088 }
1089
1090 sub status {
1091     my $cfg = read_cron();
1092
1093     my $status_list = sprintf("%-25s%-25s%-10s\n", "SOURCE", "NAME", "STATUS");
1094
1095     my $states = read_state();
1096
1097     foreach my $source (sort keys%{$cfg}) {
1098         foreach my $sync_name (sort keys%{$cfg->{$source}}) {
1099             $status_list .= sprintf("%-25s", cut_target_width($source, 25));
1100             $status_list .= sprintf("%-25s", cut_target_width($sync_name, 25));
1101             $status_list .= "$states->{$source}->{$sync_name}->{state}\n";
1102         }
1103     }
1104
1105     return $status_list;
1106 }
1107
1108 sub enable_job {
1109     my ($param) = @_;
1110
1111     locked("$CONFIG_PATH/cron_and_state.lock", sub {
1112         my $job = get_job($param);
1113         $job->{state} = "ok";
1114         update_state($job);
1115         update_cron($job);
1116     });
1117 }
1118
1119 sub disable_job {
1120     my ($param) = @_;
1121
1122     locked("$CONFIG_PATH/cron_and_state.lock", sub {
1123         my $job = get_job($param);
1124         $job->{state} = "stopped";
1125         update_state($job);
1126         update_cron($job);
1127     });
1128 }
1129
1130 my $cmd_help = {
1131     destroy => qq{
1132 $PROGNAME destroy --source <string> [OPTIONS]
1133
1134     Remove a sync Job from the scheduler
1135
1136         --name      string
1137                 The name of the sync job, if not set 'default' is used.
1138
1139         --source    string
1140                 The source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1141     },
1142     create => qq{
1143 $PROGNAME create --dest <string> --source <string> [OPTIONS]
1144
1145     Create a new sync-job
1146
1147         --dest      string
1148                 The destination target is like [IP]:<Pool>[/Path]
1149
1150         --dest-user string
1151                 The name of the user on the destination target, root by default
1152
1153         --limit     integer
1154                 Maximal sync speed in kBytes/s, default is unlimited
1155
1156         --maxsnap   integer
1157                 The number of snapshots to keep until older ones are erased.
1158                 The default is 1, use 0 for unlimited.
1159
1160         --name      string
1161                 The name of the sync job, if not set it is default
1162
1163         --prepend-storage-id
1164                 If specified, prepend the storage ID to the destination's path(s).
1165
1166         --skip
1167                 If specified, skip the first sync.
1168
1169         --source    string
1170                 The source can be an <VMID> or [IP:]<ZFSPool>[/Path]
1171
1172         --source-user    string
1173                 The (ssh) user-name on the source target, root by default
1174
1175         --properties
1176                 If specified, include the dataset's properties in the stream.
1177
1178         --dest-config-path    string
1179                 Specifies a custom config path on the destination target.
1180                 The default is /var/lib/pve-zsync
1181     },
1182     sync => qq{
1183 $PROGNAME sync --dest <string> --source <string> [OPTIONS]\n
1184
1185     Trigger one sync.
1186
1187         --dest      string
1188                 The destination target is like [IP:]<Pool>[/Path]
1189
1190         --dest-user string
1191                 The (ssh) user-name on the destination target, root by default
1192
1193         --limit     integer
1194                 The maximal sync speed in kBytes/s, default is unlimited
1195
1196         --maxsnap   integer
1197                 The number of snapshots to keep until older ones are erased.
1198                 The default is 1, use 0 for unlimited.
1199
1200         --name      string
1201                 The name of the sync job, if not set it is 'default'.
1202                 It is only necessary if scheduler allready contains this source.
1203
1204         --prepend-storage-id
1205                 If specified, prepend the storage ID to the destination's path(s).
1206
1207         --source    string
1208                 The source can either be an <VMID> or [IP:]<ZFSPool>[/Path]
1209
1210         --source-user    string
1211                 The name of the user on the source target, root by default
1212
1213         --verbose
1214                 If specified, print out the sync progress.
1215
1216         --properties
1217                 If specified, include the dataset's properties in the stream.
1218
1219         --dest-config-path    string
1220                 Specifies a custom config path on the destination target.
1221                 The default is /var/lib/pve-zsync
1222     },
1223     list => qq{
1224 $PROGNAME list
1225
1226         Get a List of all scheduled Sync Jobs
1227     },
1228     status => qq{
1229 $PROGNAME status
1230
1231         Get the status of all scheduled Sync Jobs
1232     },
1233     help => qq{
1234 $PROGNAME help <cmd> [OPTIONS]
1235
1236     Get help about specified command.
1237
1238         <cmd>      string
1239                 Command name to get help about.
1240
1241         --verbose
1242                 Verbose output format.
1243     },
1244     enable => qq{
1245 $PROGNAME enable --source <string> [OPTIONS]
1246
1247     Enable a sync-job and reset all job-errors, if any.
1248
1249         --name      string
1250                 name of the sync job, if not set it is default
1251
1252         --source    string
1253                 the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1254     },
1255     disable => qq{
1256 $PROGNAME disable --source <string> [OPTIONS]
1257
1258     Disables (pauses) a sync-job
1259
1260         --name      string
1261                 name of the sync-job, if not set it is default
1262
1263         --source    string
1264                 the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1265     },
1266     printpod => "$PROGNAME printpod\n\n\tinternal command",
1267
1268 };
1269
1270 if (!$command) {
1271     usage(); die "\n";
1272 } elsif (!$cmd_help->{$command}) {
1273     print "ERROR: unknown command '$command'";
1274     usage(1); die "\n";
1275 }
1276
1277 my @arg = @ARGV;
1278 my $param = parse_argv(@arg);
1279
1280 sub check_params {
1281     for (@_) {
1282         die "$cmd_help->{$command}\n" if !$param->{$_};
1283     }
1284 }
1285
1286 if ($command eq 'destroy') {
1287     check_params(qw(source));
1288
1289     check_target($param->{source});
1290     destroy_job($param);
1291
1292 } elsif ($command eq 'sync') {
1293     check_params(qw(source dest));
1294
1295     check_target($param->{source});
1296     check_target($param->{dest});
1297     sync($param);
1298
1299 } elsif ($command eq 'create') {
1300     check_params(qw(source dest));
1301
1302     check_target($param->{source});
1303     check_target($param->{dest});
1304     init($param);
1305
1306 } elsif ($command eq 'status') {
1307     print status();
1308
1309 } elsif ($command eq 'list') {
1310     print list();
1311
1312 } elsif ($command eq 'help') {
1313     my $help_command = $ARGV[1];
1314
1315     if ($help_command && $cmd_help->{$help_command}) {
1316         die "$cmd_help->{$help_command}\n";
1317
1318     }
1319     if ($param->{verbose}) {
1320         exec("man $PROGNAME");
1321
1322     } else {
1323         usage(1);
1324
1325     }
1326
1327 } elsif ($command eq 'enable') {
1328     check_params(qw(source));
1329
1330     check_target($param->{source});
1331     enable_job($param);
1332
1333 } elsif ($command eq 'disable') {
1334     check_params(qw(source));
1335
1336     check_target($param->{source});
1337     disable_job($param);
1338
1339 } elsif ($command eq 'printpod') {
1340     print_pod();
1341 }
1342
1343 sub usage {
1344     my ($help) = @_;
1345
1346     print("ERROR:\tno command specified\n") if !$help;
1347     print("USAGE:\t$PROGNAME <COMMAND> [ARGS] [OPTIONS]\n");
1348     print("\t$PROGNAME help [<cmd>] [OPTIONS]\n\n");
1349     print("\t$PROGNAME create --dest <string> --source <string> [OPTIONS]\n");
1350     print("\t$PROGNAME destroy --source <string> [OPTIONS]\n");
1351     print("\t$PROGNAME disable --source <string> [OPTIONS]\n");
1352     print("\t$PROGNAME enable --source <string> [OPTIONS]\n");
1353     print("\t$PROGNAME list\n");
1354     print("\t$PROGNAME status\n");
1355     print("\t$PROGNAME sync --dest <string> --source <string> [OPTIONS]\n");
1356 }
1357
1358 sub check_target {
1359     my ($target) = @_;
1360     parse_target($target);
1361 }
1362
1363 sub print_pod {
1364
1365     my $synopsis = join("\n", sort values %$cmd_help);
1366     my $commands = join(", ", sort keys %$cmd_help);
1367
1368     print <<EOF;
1369 =head1 NAME
1370
1371 pve-zsync - PVE ZFS Storage Sync Tool
1372
1373 =head1 SYNOPSIS
1374
1375 pve-zsync <COMMAND> [ARGS] [OPTIONS]
1376
1377 Where <COMMAND> can be one of: $commands
1378
1379 =head1 DESCRIPTION
1380
1381 The pve-zsync tool can help you to sync your VMs or directories stored on ZFS
1382 between multiple servers.
1383
1384 pve-zsync is able to automatically configure CRON jobs, so that a periodic sync
1385 will be automatically triggered.
1386 The default sync interval is 15 min, if you want to change this value you can
1387 do this in F</etc/cron.d/pve-zsync>. If you need help to configure CRON tabs, see
1388 man crontab.
1389
1390 =head1 COMMANDS AND OPTIONS
1391
1392 $synopsis
1393
1394 =head1 EXAMPLES
1395
1396 Adds a job for syncing the local VM 100 to a remote server's ZFS pool named "tank":
1397     pve-zsync create --source=100 -dest=192.168.1.2:tank
1398
1399 =head1 IMPORTANT FILES
1400
1401 Cron jobs and config are stored in F</etc/cron.d/pve-zsync>
1402
1403 The VM configuration itself gets copied to the destination machines
1404 F</var/lib/pve-zsync/> path.
1405
1406 =head1 COPYRIGHT AND DISCLAIMER
1407
1408 Copyright (C) 2007-2021 Proxmox Server Solutions GmbH
1409
1410 This program is free software: you can redistribute it and/or modify it under
1411 the terms of the GNU Affero General Public License as published by the Free
1412 Software Foundation, either version 3 of the License, or (at your option) any
1413 later version.
1414
1415 This program is distributed in the hope that it will be useful, but WITHOUT ANY
1416 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
1417 PARTICULAR PURPOSE. See the GNU Affero General Public License for more
1418 details.
1419
1420 You should have received a copy of the GNU Affero General Public License along
1421 with this program. If not, see <http://www.gnu.org/licenses/>.
1422
1423 EOF
1424 }