pve-zsync

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl qw(:flock SEEK_END);
   7 use Getopt::Long qw(GetOptionsFromArray);
   8 use File::Path qw(make_path);
   9 use JSON;
  10 use IO::File;
  11 use String::ShellQuote 'shell_quote';
  12
  13 my $PROGNAME = "pve-zsync";
  14 my $CONFIG_PATH = "/var/lib/${PROGNAME}";
  15 my $STATE = "${CONFIG_PATH}/sync_state";
  16 my $CRONJOBS = "/etc/cron.d/$PROGNAME";
  17 my $PATH = "/usr/sbin";
  18 my $PVE_DIR = "/etc/pve/local";
  19 my $QEMU_CONF = "${PVE_DIR}/qemu-server";
  20 my $LXC_CONF = "${PVE_DIR}/lxc";
  21 my $PROG_PATH = "$PATH/${PROGNAME}";
  22 my $INTERVAL = 15;
  23 my $DEBUG;
  24
  25 BEGIN {
  26     $DEBUG = 0; # change default here. not above on declaration!
  27     $DEBUG ||= $ENV{ZSYNC_DEBUG};
  28     if ($DEBUG) {
  29         require Data::Dumper;
  30         Data::Dumper->import();
  31     }
  32 }
  33
  34 my $IPV4OCTET = "(?:25[0-5]|(?:[1-9]|1[0-9]|2[0-4])?[0-9])";
  35 my $IPV4RE = "(?:(?:$IPV4OCTET\\.){3}$IPV4OCTET)";
  36 my $IPV6H16 = "(?:[0-9a-fA-F]{1,4})";
  37 my $IPV6LS32 = "(?:(?:$IPV4RE|$IPV6H16:$IPV6H16))";
  38
  39 my $IPV6RE = "(?:" .
  40     "(?:(?:" .                             "(?:$IPV6H16:){6})$IPV6LS32)|" .
  41     "(?:(?:" .                           "::(?:$IPV6H16:){5})$IPV6LS32)|" .
  42     "(?:(?:(?:" .              "$IPV6H16)?::(?:$IPV6H16:){4})$IPV6LS32)|" .
  43     "(?:(?:(?:(?:$IPV6H16:){0,1}$IPV6H16)?::(?:$IPV6H16:){3})$IPV6LS32)|" .
  44     "(?:(?:(?:(?:$IPV6H16:){0,2}$IPV6H16)?::(?:$IPV6H16:){2})$IPV6LS32)|" .
  45     "(?:(?:(?:(?:$IPV6H16:){0,3}$IPV6H16)?::(?:$IPV6H16:){1})$IPV6LS32)|" .
  46     "(?:(?:(?:(?:$IPV6H16:){0,4}$IPV6H16)?::" .           ")$IPV6LS32)|" .
  47     "(?:(?:(?:(?:$IPV6H16:){0,5}$IPV6H16)?::" .            ")$IPV6H16)|" .
  48     "(?:(?:(?:(?:$IPV6H16:){0,6}$IPV6H16)?::" .                    ")))";
  49
  50 my $HOSTv4RE0 = "(?:[\\w\\.\\-_]+|$IPV4RE)";       # hostname or ipv4 address
  51 my $HOSTv4RE1 = "(?:$HOSTv4RE0|\\[$HOSTv4RE0\\])"; # these may be in brackets, too
  52 my $HOSTRE = "(?:$HOSTv4RE1|\\[$IPV6RE\\])";       # ipv6 must always be in brackets
  53 # targets are either a VMID, or a 'host:zpool/path' with 'host:' being optional
  54 my $TARGETRE = qr!^(?:($HOSTRE):)?(\d+|(?:[\w\-_]+)(/.+)?)$!;
  55
  56 my $DISK_KEY_RE = qr/^(?:(?:(?:virtio|ide|scsi|sata|efidisk|mp)\d+)|rootfs): /;
  57
  58 my $INSTANCE_ID = get_instance_id($$);
  59
  60 my $command = $ARGV[0];
  61
  62 if (defined($command) && $command ne 'help' && $command ne 'printpod') {
  63     check_bin ('cstream');
  64     check_bin ('zfs');
  65     check_bin ('ssh');
  66     check_bin ('scp');
  67 }
  68
  69 $SIG{TERM} = $SIG{QUIT} = $SIG{PIPE} = $SIG{HUP} = $SIG{KILL} = $SIG{INT} =  sub {
  70     die "Signaled, aborting sync: $!\n";
  71 };
  72
  73 sub check_bin {
  74     my ($bin)  = @_;
  75
  76     foreach my $p (split (/:/, $ENV{PATH})) {
  77         my $fn = "$p/$bin";
  78         if (-x $fn) {
  79             return $fn;
  80         }
  81     }
  82
  83     die "unable to find command '$bin'\n";
  84 }
  85
  86 sub read_file {
  87     my ($filename, $one_line_only) = @_;
  88
  89     my $fh = IO::File->new($filename, "r")
  90         or die "Could not open file ${filename}: $!\n";
  91
  92     my $text = $one_line_only ? <$fh> : [ <$fh> ];
  93
  94     close($fh);
  95
  96     return $text;
  97 }
  98
  99 sub cut_target_width {
 100     my ($path, $maxlen) = @_;
 101     $path =~ s@/+@/@g;
 102
 103     return $path if length($path) <= $maxlen;
 104
 105     return '..'.substr($path, -$maxlen+2) if $path !~ m@/@;
 106
 107     $path =~ s@/([^/]+/?)$@@;
 108     my $tail = $1;
 109
 110     if (length($tail)+3 == $maxlen) {
 111         return "../$tail";
 112     } elsif (length($tail)+2 >= $maxlen) {
 113         return '..'.substr($tail, -$maxlen+2)
 114     }
 115
 116     $path =~ s@(/[^/]+)(?:/|$)@@;
 117     my $head = $1;
 118     my $both = length($head) + length($tail);
 119     my $remaining = $maxlen-$both-4; # -4 for "/../"
 120
 121     if ($remaining < 0) {
 122         return substr($head, 0, $maxlen - length($tail) - 3) . "../$tail"; # -3 for "../"
 123     }
 124
 125     substr($path, ($remaining/2), (length($path)-$remaining), '..');
 126     return "$head/" . $path . "/$tail";
 127 }
 128
 129 sub locked {
 130     my ($lock_fn, $code) = @_;
 131
 132     my $lock_fh = IO::File->new("> $lock_fn");
 133
 134     flock($lock_fh, LOCK_EX) || die "Couldn't acquire lock - $!\n";
 135     my $res = eval { $code->() };
 136     my $err = $@;
 137
 138     flock($lock_fh, LOCK_UN) || warn "Error unlocking - $!\n";
 139     die "$err" if $err;
 140
 141     close($lock_fh);
 142     return $res;
 143 }
 144
 145 sub get_status {
 146     my ($source, $name, $status) = @_;
 147
 148     if ($status->{$source->{all}}->{$name}->{status}) {
 149         return $status;
 150     }
 151
 152     return undef;
 153 }
 154
 155 sub check_dataset_exists {
 156     my ($dataset, $ip, $user) = @_;
 157
 158     my $cmd = [];
 159
 160     if ($ip) {
 161         push @$cmd, 'ssh', "$user\@$ip", '--';
 162     }
 163     push @$cmd, 'zfs', 'list', '-H', '--', $dataset;
 164     eval {
 165         run_cmd($cmd);
 166     };
 167
 168     if ($@) {
 169         return 0;
 170     }
 171     return 1;
 172 }
 173
 174 sub create_file_system {
 175     my ($file_system, $ip, $user) = @_;
 176
 177     my $cmd = [];
 178
 179     if ($ip) {
 180         push @$cmd, 'ssh', "$user\@$ip", '--';
 181     }
 182     push @$cmd, 'zfs', 'create', $file_system;
 183
 184     run_cmd($cmd);
 185 }
 186
 187 sub parse_target {
 188     my ($text) = @_;
 189
 190     my $errstr = "$text : is not a valid input! Use [IP:]<VMID> or [IP:]<ZFSPool>[/Path]";
 191     my $target = {};
 192
 193     if ($text !~ $TARGETRE) {
 194         die "$errstr\n";
 195     }
 196     $target->{all} = $2;
 197     $target->{ip} = $1 if $1;
 198     my @parts = split('/', $2);
 199
 200     $target->{ip} =~ s/^\[(.*)\]$/$1/ if $target->{ip};
 201
 202     my $pool = $target->{pool} = shift(@parts);
 203     die "$errstr\n" if !$pool;
 204
 205     if ($pool =~ m/^\d+$/) {
 206         $target->{vmid} = $pool;
 207         delete $target->{pool};
 208     }
 209
 210     return $target if (@parts == 0);
 211     $target->{last_part} = pop(@parts);
 212
 213     if ($target->{ip}) {
 214         pop(@parts);
 215     }
 216     if (@parts > 0) {
 217         $target->{path} = join('/', @parts);
 218     }
 219
 220     return $target;
 221 }
 222
 223 sub read_cron {
 224
 225     #This is for the first use to init file;
 226     if (!-e $CRONJOBS) {
 227         my $new_fh = IO::File->new("> $CRONJOBS");
 228         die "Could not create $CRONJOBS: $!\n" if !$new_fh;
 229         close($new_fh);
 230         return undef;
 231     }
 232
 233     my $text = read_file($CRONJOBS, 0);
 234
 235     return encode_cron(@{$text});
 236 }
 237
 238 sub parse_argv {
 239     my (@arg) = @_;
 240
 241     my $param = {
 242         dest => undef,
 243         source => undef,
 244         verbose => undef,
 245         limit => undef,
 246         maxsnap => undef,
 247         dest_maxsnap => undef,
 248         name => undef,
 249         skip => undef,
 250         method => undef,
 251         source_user => undef,
 252         dest_user => undef,
 253         prepend_storage_id => undef,
 254         properties => undef,
 255         dest_config_path => undef,
 256     };
 257
 258     my ($ret) = GetOptionsFromArray(
 259         \@arg,
 260         'dest=s' => \$param->{dest},
 261         'source=s' => \$param->{source},
 262         'verbose' => \$param->{verbose},
 263         'limit=i' => \$param->{limit},
 264         'maxsnap=i' => \$param->{maxsnap},
 265         'dest-maxsnap=i' => \$param->{dest_maxsnap},
 266         'name=s' => \$param->{name},
 267         'skip' => \$param->{skip},
 268         'method=s' => \$param->{method},
 269         'source-user=s' => \$param->{source_user},
 270         'dest-user=s' => \$param->{dest_user},
 271         'prepend-storage-id' => \$param->{prepend_storage_id},
 272         'properties' => \$param->{properties},
 273         'dest-config-path=s' => \$param->{dest_config_path},
 274     );
 275
 276     die "can't parse options\n" if $ret == 0;
 277
 278     $param->{name} //= "default";
 279     $param->{maxsnap} //= 1;
 280     $param->{method} //= "ssh";
 281     $param->{source_user} //= "root";
 282     $param->{dest_user} //= "root";
 283
 284     return $param;
 285 }
 286
 287 sub add_state_to_job {
 288     my ($job) = @_;
 289
 290     my $states = read_state();
 291     my $state = $states->{$job->{source}}->{$job->{name}};
 292
 293     $job->{state} = $state->{state};
 294     $job->{lsync} = $state->{lsync};
 295     $job->{vm_type} = $state->{vm_type};
 296     $job->{instance_id} = $state->{instance_id};
 297
 298     for (my $i = 0; $state->{"snap$i"}; $i++) {
 299         $job->{"snap$i"} = $state->{"snap$i"};
 300     }
 301
 302     return $job;
 303 }
 304
 305 sub encode_cron {
 306     my (@text) = @_;
 307
 308     my $cfg = {};
 309
 310     while (my $line = shift(@text)) {
 311
 312         my @arg = split('\s', $line);
 313         my $param = parse_argv(@arg);
 314
 315         if ($param->{source} && $param->{dest}) {
 316             my $source = delete $param->{source};
 317             my $name = delete $param->{name};
 318
 319             $cfg->{$source}->{$name} = $param;
 320         }
 321     }
 322
 323     return $cfg;
 324 }
 325
 326 sub param_to_job {
 327     my ($param) = @_;
 328
 329     my $job = {};
 330
 331     my $source = parse_target($param->{source});
 332     my $dest;
 333     $dest = parse_target($param->{dest}) if $param->{dest};
 334
 335     $job->{name} = !$param->{name} ? "default" : $param->{name};
 336     $job->{dest} = $param->{dest} if $param->{dest};
 337     $job->{method} = "local" if !$dest->{ip} && !$source->{ip};
 338     $job->{method} = "ssh" if !$job->{method};
 339     $job->{limit} = $param->{limit};
 340     $job->{maxsnap} = $param->{maxsnap};
 341     $job->{dest_maxsnap} = $param->{dest_maxsnap};
 342     $job->{source} = $param->{source};
 343     $job->{source_user} = $param->{source_user};
 344     $job->{dest_user} = $param->{dest_user};
 345     $job->{prepend_storage_id} = !!$param->{prepend_storage_id};
 346     $job->{properties} = !!$param->{properties};
 347     $job->{dest_config_path} = $param->{dest_config_path} if $param->{dest_config_path};
 348
 349     return $job;
 350 }
 351
 352 sub read_state {
 353
 354     if (!-e $STATE) {
 355         make_path $CONFIG_PATH;
 356         my $new_fh = IO::File->new("> $STATE");
 357         die "Could not create $STATE: $!\n" if !$new_fh;
 358         print $new_fh "{}";
 359         close($new_fh);
 360         return undef;
 361     }
 362
 363     my $text = read_file($STATE, 1);
 364     return decode_json($text);
 365 }
 366
 367 sub update_state {
 368     my ($job) = @_;
 369
 370     my $text = eval { read_file($STATE, 1); };
 371
 372     my $out_fh = IO::File->new("> $STATE.new");
 373     die "Could not open file ${STATE}.new: $!\n" if !$out_fh;
 374
 375     my $states = {};
 376     my $state = {};
 377     if ($text){
 378         $states = decode_json($text);
 379         $state = $states->{$job->{source}}->{$job->{name}};
 380     }
 381
 382     if ($job->{state} ne "del") {
 383         $state->{state} = $job->{state};
 384         $state->{lsync} = $job->{lsync};
 385         $state->{instance_id} = $job->{instance_id};
 386         $state->{vm_type} = $job->{vm_type};
 387
 388         for (my $i = 0; $job->{"snap$i"} ; $i++) {
 389             $state->{"snap$i"} = $job->{"snap$i"};
 390         }
 391         $states->{$job->{source}}->{$job->{name}} = $state;
 392     } else {
 393
 394         delete $states->{$job->{source}}->{$job->{name}};
 395         delete $states->{$job->{source}} if !keys %{$states->{$job->{source}}};
 396     }
 397
 398     $text = encode_json($states);
 399     print $out_fh $text;
 400
 401     close($out_fh);
 402     rename "$STATE.new", $STATE;
 403
 404     return $states;
 405 }
 406
 407 sub update_cron {
 408     my ($job) = @_;
 409
 410     my $updated;
 411     my $has_header;
 412     my $line_no = 0;
 413     my $text = "";
 414     my $header = "SHELL=/bin/sh\n";
 415     $header .= "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n\n";
 416
 417     my $current = read_file($CRONJOBS, 0);
 418
 419     foreach my $line (@{$current}) {
 420         chomp($line);
 421         if ($line =~ m/source $job->{source} .*name $job->{name} /) {
 422             $updated = 1;
 423             next if $job->{state} eq "del";
 424             $text .= format_job($job, $line);
 425         } else {
 426             if (($line_no < 3) && ($line =~ /^(PATH|SHELL)/ )) {
 427                 $has_header = 1;
 428             }
 429             $text .= "$line\n";
 430         }
 431         $line_no++;
 432     }
 433
 434     if (!$has_header) {
 435         $text = "$header$text";
 436     }
 437
 438     if (!$updated) {
 439         $text .= format_job($job);
 440     }
 441     my $new_fh = IO::File->new("> ${CRONJOBS}.new");
 442     die "Could not open file ${CRONJOBS}.new: $!\n" if !$new_fh;
 443
 444     print $new_fh $text or die "can't write to $CRONJOBS.new: $!\n";
 445     close ($new_fh);
 446
 447     rename "${CRONJOBS}.new", $CRONJOBS or die "can't move $CRONJOBS.new: $!\n";
 448 }
 449
 450 sub format_job {
 451     my ($job, $line) = @_;
 452     my $text = "";
 453
 454     if ($job->{state} eq "stopped") {
 455         $text = "#";
 456     }
 457     if ($line) {
 458         $line =~ /^#*\s*((?:\S+\s+){4}\S+)\s+root/;
 459         $text .= $1;
 460     } else {
 461         $text .= "*/$INTERVAL * * * *";
 462     }
 463     $text .= " root";
 464     $text .= " $PROGNAME sync --source $job->{source} --dest $job->{dest}";
 465     $text .= " --name $job->{name} --maxsnap $job->{maxsnap}";
 466     $text .= " --dest-maxsnap $job->{dest_maxsnap}" if defined($job->{dest_maxsnap});
 467     $text .= " --limit $job->{limit}" if $job->{limit};
 468     $text .= " --method $job->{method}";
 469     $text .= " --verbose" if $job->{verbose};
 470     $text .= " --source-user $job->{source_user}";
 471     $text .= " --dest-user $job->{dest_user}";
 472     $text .= " --prepend-storage-id" if $job->{prepend_storage_id};
 473     $text .= " --properties" if $job->{properties};
 474     $text .= " --dest-config-path $job->{dest_config_path}" if $job->{dest_config_path};
 475     $text .= "\n";
 476
 477     return $text;
 478 }
 479
 480 sub list {
 481
 482     my $cfg = read_cron();
 483
 484     my $list = sprintf("%-25s%-25s%-10s%-20s%-6s%-5s\n" , "SOURCE", "NAME", "STATE", "LAST SYNC", "TYPE", "CON");
 485
 486     my $states = read_state();
 487     foreach my $source (sort keys%{$cfg}) {
 488         foreach my $name (sort keys%{$cfg->{$source}}) {
 489             $list .= sprintf("%-25s", cut_target_width($source, 25));
 490             $list .= sprintf("%-25s", cut_target_width($name, 25));
 491             $list .= sprintf("%-10s", $states->{$source}->{$name}->{state});
 492             $list .= sprintf("%-20s", $states->{$source}->{$name}->{lsync});
 493             $list .= sprintf("%-6s", defined($states->{$source}->{$name}->{vm_type}) ? $states->{$source}->{$name}->{vm_type} : "undef");
 494             $list .= sprintf("%-5s\n", $cfg->{$source}->{$name}->{method});
 495         }
 496     }
 497
 498     return $list;
 499 }
 500
 501 sub vm_exists {
 502     my ($target, $user) = @_;
 503
 504     return undef if !defined($target->{vmid});
 505
 506     my $conf_fn = "$target->{vmid}.conf";
 507
 508     if ($target->{ip}) {
 509         my @cmd = ('ssh', "$user\@$target->{ip}", '--', '/bin/ls');
 510         return "qemu" if eval { run_cmd([@cmd, "$QEMU_CONF/$conf_fn"]) };
 511         return "lxc" if  eval { run_cmd([@cmd, "$LXC_CONF/$conf_fn"]) };
 512     } else {
 513         return "qemu" if -f "$QEMU_CONF/$conf_fn";
 514         return "lxc" if -f "$LXC_CONF/$conf_fn";
 515     }
 516
 517     return undef;
 518 }
 519
 520 sub init {
 521     my ($param) = @_;
 522
 523     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 524         my $cfg = read_cron();
 525
 526         my $job = param_to_job($param);
 527
 528         $job->{state} = "ok";
 529         $job->{lsync} = 0;
 530
 531         my $source = parse_target($param->{source});
 532         my $dest = parse_target($param->{dest});
 533
 534         if (my $ip =  $dest->{ip}) {
 535             run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{dest_user}\@$ip"]);
 536         }
 537
 538         if (my $ip =  $source->{ip}) {
 539             run_cmd(['ssh-copy-id', '-i', '/root/.ssh/id_rsa.pub', "$param->{source_user}\@$ip"]);
 540         }
 541
 542         die "Pool $dest->{all} does not exist\n"
 543             if !check_dataset_exists($dest->{all}, $dest->{ip}, $param->{dest_user});
 544
 545         if (!defined($source->{vmid})) {
 546             die "Pool $source->{all} does not exist\n"
 547                 if !check_dataset_exists($source->{all}, $source->{ip}, $param->{source_user});
 548         }
 549
 550         my $vm_type = vm_exists($source, $param->{source_user});
 551         $job->{vm_type} = $vm_type;
 552         $source->{vm_type} = $vm_type;
 553
 554         die "VM $source->{vmid} doesn't exist\n" if $source->{vmid} && !$vm_type;
 555
 556         die "Config already exists\n" if $cfg->{$job->{source}}->{$job->{name}};
 557
 558         #check if vm has zfs disks if not die;
 559         get_disks($source, $param->{source_user}) if $source->{vmid};
 560
 561         update_cron($job);
 562         update_state($job);
 563     }); #cron and state lock
 564
 565     return if $param->{skip};
 566
 567     eval { sync($param) };
 568     if (my $err = $@) {
 569         destroy_job($param);
 570         print $err;
 571     }
 572 }
 573
 574 sub get_job {
 575     my ($param) = @_;
 576
 577     my $cfg = read_cron();
 578
 579     if (!$cfg->{$param->{source}}->{$param->{name}}) {
 580         die "Job  with source $param->{source} and name $param->{name} does not exist\n" ;
 581     }
 582     my $job = $cfg->{$param->{source}}->{$param->{name}};
 583     $job->{name} = $param->{name};
 584     $job->{source} = $param->{source};
 585     $job = add_state_to_job($job);
 586
 587     return $job;
 588 }
 589
 590 sub destroy_job {
 591     my ($param) = @_;
 592
 593     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 594         my $job = get_job($param);
 595         $job->{state} = "del";
 596
 597         update_cron($job);
 598         update_state($job);
 599     });
 600 }
 601
 602 sub get_instance_id {
 603     my ($pid) = @_;
 604
 605     my $stat = read_file("/proc/$pid/stat", 1)
 606         or die "unable to read process stats\n";
 607     my $boot_id = read_file("/proc/sys/kernel/random/boot_id", 1)
 608         or die "unable to read boot ID\n";
 609
 610     my $stats = [ split(/\s+/, $stat) ];
 611     my $starttime = $stats->[21];
 612     chomp($boot_id);
 613
 614     return "${pid}:${starttime}:${boot_id}";
 615 }
 616
 617 sub instance_exists {
 618     my ($instance_id) = @_;
 619
 620     if (defined($instance_id) && $instance_id =~ m/^([1-9][0-9]*):/) {
 621         my $pid = $1;
 622         my $actual_id = eval { get_instance_id($pid); };
 623         return defined($actual_id) && $actual_id eq $instance_id;
 624     }
 625
 626     return 0;
 627 }
 628
 629 sub sync {
 630     my ($param) = @_;
 631
 632     my $job;
 633
 634     locked("$CONFIG_PATH/cron_and_state.lock", sub {
 635         eval { $job = get_job($param) };
 636
 637         if ($job) {
 638             my $state = $job->{state} // 'ok';
 639             $state = 'ok' if !instance_exists($job->{instance_id});
 640
 641             if ($state eq "syncing" || $state eq "waiting") {
 642                 die "Job --source $param->{source} --name $param->{name} is already scheduled to sync\n";
 643             }
 644
 645             $job->{state} = "waiting";
 646             $job->{instance_id} = $INSTANCE_ID;
 647
 648             update_state($job);
 649         }
 650     });
 651
 652     locked("$CONFIG_PATH/sync.lock", sub {
 653
 654         my $date = get_date();
 655
 656         my $dest;
 657         my $source;
 658         my $vm_type;
 659
 660         locked("$CONFIG_PATH/cron_and_state.lock", sub {
 661             #job might've changed while we waited for the sync lock, but we can be sure it's not syncing
 662             eval { $job = get_job($param); };
 663
 664             if ($job && defined($job->{state}) && $job->{state} eq "stopped") {
 665                 die "Job --source $param->{source} --name $param->{name} has been disabled\n";
 666             }
 667
 668             $dest = parse_target($param->{dest});
 669             $source = parse_target($param->{source});
 670
 671             $vm_type = vm_exists($source, $param->{source_user});
 672             $source->{vm_type} = $vm_type;
 673
 674             if ($job) {
 675                 $job->{state} = "syncing";
 676                 $job->{vm_type} = $vm_type if !$job->{vm_type};
 677                 update_state($job);
 678             }
 679         }); #cron and state lock
 680
 681         my $sync_path = sub {
 682             my ($source, $dest, $job, $param, $date) = @_;
 683
 684             my $dest_dataset = target_dataset($source, $dest);
 685
 686             ($dest->{old_snap}, $dest->{last_snap}) = snapshot_get(
 687                 $dest_dataset,
 688                 $param->{dest_maxsnap} // $param->{maxsnap},
 689                 $param->{name},
 690                 $dest->{ip},
 691                 $param->{dest_user},
 692             );
 693
 694             ($source->{old_snap}) = snapshot_get(
 695                 $source->{all},
 696                 $param->{maxsnap},
 697                 $param->{name},
 698                 $source->{ip},
 699                 $param->{source_user},
 700             );
 701
 702             prepare_prepended_target($source, $dest, $param->{dest_user}) if defined($dest->{prepend});
 703
 704             snapshot_add($source, $dest, $param->{name}, $date, $param->{source_user}, $param->{dest_user});
 705
 706             send_image($source, $dest, $param);
 707
 708             for my $old_snap (@{$source->{old_snap}}) {
 709                 snapshot_destroy($source->{all}, $old_snap, $source->{ip}, $param->{source_user});
 710             }
 711
 712             for my $old_snap (@{$dest->{old_snap}}) {
 713                 snapshot_destroy($dest_dataset, $old_snap, $dest->{ip}, $param->{dest_user});
 714             }
 715         };
 716
 717         eval{
 718             if ($source->{vmid}) {
 719                 die "VM $source->{vmid} doesn't exist\n" if !$vm_type;
 720                 die "source-user has to be root for syncing VMs\n" if ($param->{source_user} ne "root");
 721                 my $disks = get_disks($source, $param->{source_user});
 722
 723                 foreach my $disk (sort keys %{$disks}) {
 724                     $source->{all} = $disks->{$disk}->{all};
 725                     $source->{pool} = $disks->{$disk}->{pool};
 726                     $source->{path} = $disks->{$disk}->{path} if $disks->{$disk}->{path};
 727                     $source->{last_part} = $disks->{$disk}->{last_part};
 728
 729                     $dest->{prepend} = $disks->{$disk}->{storage_id}
 730                         if $param->{prepend_storage_id};
 731
 732                     &$sync_path($source, $dest, $job, $param, $date);
 733                 }
 734                 if ($param->{method} eq "ssh" && ($source->{ip} || $dest->{ip})) {
 735                     send_config($source, $dest,'ssh', $param->{source_user}, $param->{dest_user}, $param->{dest_config_path});
 736                 } else {
 737                     send_config($source, $dest,'local', $param->{source_user}, $param->{dest_user}, $param->{dest_config_path});
 738                 }
 739             } else {
 740                 &$sync_path($source, $dest, $job, $param, $date);
 741             }
 742         };
 743         if (my $err = $@) {
 744             locked("$CONFIG_PATH/cron_and_state.lock", sub {
 745                 eval { $job = get_job($param); };
 746                 if ($job) {
 747                     $job->{state} = "error";
 748                     delete $job->{instance_id};
 749                     update_state($job);
 750                 }
 751             });
 752             print "Job --source $param->{source} --name $param->{name} got an ERROR!!!\nERROR Message:\n";
 753             die "$err\n";
 754         }
 755
 756         locked("$CONFIG_PATH/cron_and_state.lock", sub {
 757             eval { $job = get_job($param); };
 758             if ($job) {
 759                 if (defined($job->{state}) && $job->{state} eq "stopped") {
 760                     $job->{state} = "stopped";
 761                 } else {
 762                     $job->{state} = "ok";
 763                 }
 764                 $job->{lsync} = $date;
 765                 delete $job->{instance_id};
 766                 update_state($job);
 767             }
 768         });
 769     }); #sync lock
 770 }
 771
 772 sub snapshot_get{
 773     my ($dataset, $max_snap, $name, $ip, $user) = @_;
 774
 775     my $cmd = [];
 776     push @$cmd, 'ssh', "$user\@$ip", '--', if $ip;
 777     push @$cmd, 'zfs', 'list', '-r', '-t', 'snapshot', '-Ho', 'name', '-S', 'creation';
 778     push @$cmd, $dataset;
 779
 780     my $raw;
 781     eval {$raw = run_cmd($cmd)};
 782     if (my $erro =$@) { #this means the volume doesn't exist on dest yet
 783        return undef;
 784     }
 785
 786     my $index = 0;
 787     my $line = "";
 788     my $last_snap = undef;
 789     my $old_snap = [];
 790
 791     while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
 792         $line = $1;
 793         if ($line =~ m/@(.*)$/) {
 794             $last_snap = $1 if (!$last_snap);
 795         }
 796         if ($line =~ m/(rep_\Q${name}\E_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})$/) {
 797             # interpreted as infinity
 798             last if $max_snap <= 0;
 799
 800             my $snap = $1;
 801             $index++;
 802
 803             if ($index >= $max_snap) {
 804                 push @{$old_snap}, $snap;
 805             }
 806         }
 807     }
 808
 809     return ($old_snap, $last_snap) if $last_snap;
 810
 811     return undef;
 812 }
 813
 814 sub snapshot_add {
 815     my ($source, $dest, $name, $date, $source_user, $dest_user) = @_;
 816
 817     my $snap_name = "rep_$name\_".$date;
 818
 819     $source->{new_snap} = $snap_name;
 820
 821     my $path = "$source->{all}\@$snap_name";
 822
 823     my $cmd = [];
 824     push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--', if $source->{ip};
 825     push @$cmd, 'zfs', 'snapshot', $path;
 826     eval{
 827         run_cmd($cmd);
 828     };
 829
 830     if (my $err = $@) {
 831         snapshot_destroy($source->{all}, $snap_name, $source->{ip}, $source_user);
 832         die "$err\n";
 833     }
 834 }
 835
 836 sub get_disks {
 837     my ($target, $user) = @_;
 838
 839     my $cmd = [];
 840     push @$cmd, 'ssh', "$user\@$target->{ip}", '--', if $target->{ip};
 841
 842     if ($target->{vm_type} eq 'qemu') {
 843         push @$cmd, 'qm', 'config', $target->{vmid};
 844     } elsif ($target->{vm_type} eq 'lxc') {
 845         push @$cmd, 'pct', 'config', $target->{vmid};
 846     } else {
 847         die "VM Type unknown\n";
 848     }
 849
 850     my $res = run_cmd($cmd);
 851
 852     my $disks = parse_disks($res, $target->{ip}, $target->{vm_type}, $user);
 853
 854     return $disks;
 855 }
 856
 857 sub run_cmd {
 858     my ($cmd) = @_;
 859     print "Start CMD\n" if $DEBUG;
 860     print Dumper $cmd if $DEBUG;
 861     if (ref($cmd) eq 'ARRAY') {
 862         $cmd = join(' ', map { ref($_) ? $$_ : shell_quote($_) } @$cmd);
 863     }
 864     my $output = `$cmd 2>&1`;
 865
 866     die "COMMAND:\n\t$cmd\nGET ERROR:\n\t$output" if 0 != $?;
 867
 868     chomp($output);
 869     print Dumper $output if $DEBUG;
 870     print "END CMD\n" if $DEBUG;
 871     return $output;
 872 }
 873
 874 sub parse_disks {
 875     my ($text, $ip, $vm_type, $user) = @_;
 876
 877     my $disks;
 878
 879     my $num = 0;
 880     while ($text && $text =~ s/^(.*?)(\n|$)//) {
 881         my $line = $1;
 882
 883         next if $line =~ /media=cdrom/;
 884         next if $line !~ m/$DISK_KEY_RE/;
 885
 886         #QEMU if backup is not set include in  sync
 887         next if $vm_type eq 'qemu' && ($line =~ m/backup=(?i:0|no|off|false)/);
 888
 889         #LXC if backup is not set do no in sync
 890         next if $vm_type eq 'lxc' && ($line =~ m/^mp\d:/) && ($line !~ m/backup=(?i:1|yes|on|true)/);
 891
 892         my $disk = undef;
 893         my $stor = undef;
 894         if($line =~ m/$DISK_KEY_RE(.*)$/) {
 895             my @parameter = split(/,/,$1);
 896
 897             foreach my $opt (@parameter) {
 898                 if ($opt =~ m/^(?:file=|volume=)?([^:]+):([A-Za-z0-9\-]+)$/){
 899                     $disk = $2;
 900                     $stor = $1;
 901                     last;
 902                 }
 903             }
 904         }
 905         if (!defined($disk) || !defined($stor)) {
 906             print "Disk: \"$line\" has no valid zfs dataset format and will be skipped\n";
 907             next;
 908         }
 909
 910         my $cmd = [];
 911         push @$cmd, 'ssh', "$user\@$ip", '--' if $ip;
 912         push @$cmd, 'pvesm', 'path', "$stor:$disk";
 913         my $path = run_cmd($cmd);
 914
 915         die "Get no path from pvesm path $stor:$disk\n" if !$path;
 916
 917         $disks->{$num}->{storage_id} = $stor;
 918
 919         if ($vm_type eq 'qemu' && $path =~ m/^\/dev\/zvol\/(\w+.*)(\/$disk)$/) {
 920
 921             my @array = split('/', $1);
 922             $disks->{$num}->{pool} = shift(@array);
 923             $disks->{$num}->{all} = $disks->{$num}->{pool};
 924             if (0 < @array) {
 925                 $disks->{$num}->{path} = join('/', @array);
 926                 $disks->{$num}->{all} .= "\/$disks->{$num}->{path}";
 927             }
 928             $disks->{$num}->{last_part} = $disk;
 929             $disks->{$num}->{all} .= "\/$disk";
 930
 931             $num++;
 932         } elsif ($vm_type eq 'lxc' && $path =~ m/^\/(\w+.+)(\/(\w+.*))*(\/$disk)$/) {
 933
 934             $disks->{$num}->{pool} = $1;
 935             $disks->{$num}->{all} = $disks->{$num}->{pool};
 936
 937             if ($2) {
 938                 $disks->{$num}->{path} = $3;
 939                 $disks->{$num}->{all} .= "\/$disks->{$num}->{path}";
 940             }
 941
 942             $disks->{$num}->{last_part} = $disk;
 943             $disks->{$num}->{all} .= "\/$disk";
 944
 945             $num++;
 946
 947         } else {
 948             die "ERROR: in path\n";
 949         }
 950     }
 951
 952     die "Vm include no disk on zfs.\n" if !$disks->{0};
 953     return $disks;
 954 }
 955
 956 # how the corresponding dataset is named on the target
 957 sub target_dataset {
 958     my ($source, $dest) = @_;
 959
 960     my $target = "$dest->{all}";
 961     $target .= "/$dest->{prepend}" if defined($dest->{prepend});
 962     $target .= "/$source->{last_part}" if $source->{last_part};
 963     $target =~ s!/+!/!g;
 964
 965     return $target;
 966 }
 967
 968 # create the parent dataset for the actual target
 969 sub prepare_prepended_target {
 970     my ($source, $dest, $dest_user) = @_;
 971
 972     die "internal error - not a prepended target\n" if !defined($dest->{prepend});
 973
 974     # The parent dataset shouldn't be the actual target.
 975     die "internal error - no last_part for source\n" if !$source->{last_part};
 976
 977     my $target = "$dest->{all}/$dest->{prepend}";
 978     $target =~ s!/+!/!g;
 979
 980     return if check_dataset_exists($target, $dest->{ip}, $dest_user);
 981
 982     create_file_system($target, $dest->{ip}, $dest_user);
 983 }
 984
 985 sub snapshot_destroy {
 986     my ($dataset, $snap, $ip, $user) = @_;
 987
 988     my @zfscmd = ('zfs', 'destroy');
 989     my $snapshot = "$dataset\@$snap";
 990
 991     eval {
 992         if ($ip) {
 993             run_cmd(['ssh', "$user\@$ip", '--', @zfscmd, $snapshot]);
 994         } else {
 995             run_cmd([@zfscmd, $snapshot]);
 996         }
 997     };
 998     if (my $erro = $@) {
 999         warn "WARN: $erro";
1000     }
1001 }
1002
1003 # check if snapshot for incremental sync exist on source side
1004 sub snapshot_exist {
1005     my ($source , $dest, $method, $source_user) = @_;
1006
1007     my $cmd = [];
1008     push @$cmd, 'ssh', "$source_user\@$source->{ip}", '--' if $source->{ip};
1009     push @$cmd, 'zfs', 'list', '-rt', 'snapshot', '-Ho', 'name';
1010
1011     my $path = $source->{all};
1012     $path .= "\@$dest->{last_snap}";
1013
1014     push @$cmd, $path;
1015
1016     eval {run_cmd($cmd)};
1017     if (my $erro =$@) {
1018         warn "WARN: $erro";
1019         return undef;
1020     }
1021     return 1;
1022 }
1023
1024 sub send_image {
1025     my ($source, $dest, $param) = @_;
1026
1027     my $cmd = [];
1028
1029     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{source_user}\@$source->{ip}", '--' if $source->{ip};
1030     push @$cmd, 'zfs', 'send';
1031     push @$cmd, '-p', if $param->{properties};
1032     push @$cmd, '-v' if $param->{verbose};
1033
1034     if($dest->{last_snap} && snapshot_exist($source , $dest, $param->{method}, $param->{source_user})) {
1035         push @$cmd, '-i', "$source->{all}\@$dest->{last_snap}";
1036     }
1037     push @$cmd, '--', "$source->{all}\@$source->{new_snap}";
1038
1039     if ($param->{limit}){
1040         my $bwl = $param->{limit}*1024;
1041         push @$cmd, \'|', 'cstream', '-t', $bwl;
1042     }
1043     my $target = target_dataset($source, $dest);
1044
1045     push @$cmd, \'|';
1046     push @$cmd, 'ssh', '-o', 'BatchMode=yes', "$param->{dest_user}\@$dest->{ip}", '--' if $dest->{ip};
1047     push @$cmd, 'zfs', 'recv', '-F', '--';
1048     push @$cmd, "$target";
1049
1050     eval {
1051         run_cmd($cmd)
1052     };
1053
1054     if (my $erro = $@) {
1055         snapshot_destroy($source->{all}, $source->{new_snap}, $source->{ip}, $param->{source_user});
1056         die $erro;
1057     };
1058 }
1059
1060
1061 sub send_config{
1062     my ($source, $dest, $method, $source_user, $dest_user, $dest_config_path) = @_;
1063
1064     my $source_target = $source->{vm_type} eq 'qemu' ? "$QEMU_CONF/$source->{vmid}.conf": "$LXC_CONF/$source->{vmid}.conf";
1065     my $dest_target_new ="$source->{vmid}.conf.$source->{vm_type}.$source->{new_snap}";
1066
1067     my $config_dir = $dest_config_path // $CONFIG_PATH;
1068     $config_dir .= "/$dest->{last_part}" if $dest->{last_part};
1069
1070     $dest_target_new = $config_dir.'/'.$dest_target_new;
1071
1072     if ($method eq 'ssh'){
1073         if ($dest->{ip} && $source->{ip}) {
1074             run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'mkdir', '-p', '--', $config_dir]);
1075             run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", "$dest_user\@[$dest->{ip}]:$dest_target_new"]);
1076         } elsif ($dest->{ip}) {
1077             run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'mkdir', '-p', '--', $config_dir]);
1078             run_cmd(['scp', '--', $source_target, "$dest_user\@[$dest->{ip}]:$dest_target_new"]);
1079         } elsif ($source->{ip}) {
1080             run_cmd(['mkdir', '-p', '--', $config_dir]);
1081             run_cmd(['scp', '--', "$source_user\@[$source->{ip}]:$source_target", $dest_target_new]);
1082         }
1083
1084         for my $old_snap (@{$dest->{old_snap}}) {
1085             my $dest_target_old ="${config_dir}/$source->{vmid}.conf.$source->{vm_type}.${old_snap}";
1086             if($dest->{ip}){
1087                 run_cmd(['ssh', "$dest_user\@$dest->{ip}", '--', 'rm', '-f', '--', $dest_target_old]);
1088             } else {
1089                 run_cmd(['rm', '-f', '--', $dest_target_old]);
1090             }
1091         }
1092     } elsif ($method eq 'local') {
1093         run_cmd(['mkdir', '-p', '--', $config_dir]);
1094         run_cmd(['cp', $source_target, $dest_target_new]);
1095     }
1096 }
1097
1098 sub get_date {
1099     my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
1100     my $datestamp = sprintf ("%04d-%02d-%02d_%02d:%02d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
1101
1102     return $datestamp;
1103 }
1104
1105 sub status {
1106     my $cfg = read_cron();
1107
1108     my $status_list = sprintf("%-25s%-25s%-10s\n", "SOURCE", "NAME", "STATUS");
1109
1110     my $states = read_state();
1111
1112     foreach my $source (sort keys%{$cfg}) {
1113         foreach my $sync_name (sort keys%{$cfg->{$source}}) {
1114             $status_list .= sprintf("%-25s", cut_target_width($source, 25));
1115             $status_list .= sprintf("%-25s", cut_target_width($sync_name, 25));
1116             $status_list .= "$states->{$source}->{$sync_name}->{state}\n";
1117         }
1118     }
1119
1120     return $status_list;
1121 }
1122
1123 sub enable_job {
1124     my ($param) = @_;
1125
1126     locked("$CONFIG_PATH/cron_and_state.lock", sub {
1127         my $job = get_job($param);
1128         $job->{state} = "ok";
1129         update_state($job);
1130         update_cron($job);
1131     });
1132 }
1133
1134 sub disable_job {
1135     my ($param) = @_;
1136
1137     locked("$CONFIG_PATH/cron_and_state.lock", sub {
1138         my $job = get_job($param);
1139         $job->{state} = "stopped";
1140         update_state($job);
1141         update_cron($job);
1142     });
1143 }
1144
1145 my $cmd_help = {
1146     destroy => qq{
1147 $PROGNAME destroy --source <string> [OPTIONS]
1148
1149     Remove a sync Job from the scheduler
1150
1151         --name      string
1152                 The name of the sync job, if not set 'default' is used.
1153
1154         --source    string
1155                 The source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1156     },
1157     create => qq{
1158 $PROGNAME create --dest <string> --source <string> [OPTIONS]
1159
1160     Create a new sync-job
1161
1162         --dest      string
1163                 The destination target is like [IP]:<Pool>[/Path]
1164
1165         --dest-user string
1166                 The name of the user on the destination target, root by default
1167
1168         --limit     integer
1169                 Maximal sync speed in kBytes/s, default is unlimited
1170
1171         --maxsnap   integer
1172                 The number of snapshots to keep until older ones are erased.
1173                 The default is 1, use 0 for unlimited.
1174
1175         --dest-maxsnap   integer
1176                 Override maxsnap for the destination dataset.
1177
1178         --name      string
1179                 The name of the sync job, if not set it is default
1180
1181         --prepend-storage-id
1182                 If specified, prepend the storage ID to the destination's path(s).
1183
1184         --skip
1185                 If specified, skip the first sync.
1186
1187         --source    string
1188                 The source can be an <VMID> or [IP:]<ZFSPool>[/Path]
1189
1190         --source-user    string
1191                 The (ssh) user-name on the source target, root by default
1192
1193         --properties
1194                 If specified, include the dataset's properties in the stream.
1195
1196         --dest-config-path    string
1197                 Specifies a custom config path on the destination target.
1198                 The default is /var/lib/pve-zsync
1199     },
1200     sync => qq{
1201 $PROGNAME sync --dest <string> --source <string> [OPTIONS]\n
1202
1203     Trigger one sync.
1204
1205         --dest      string
1206                 The destination target is like [IP:]<Pool>[/Path]
1207
1208         --dest-user string
1209                 The (ssh) user-name on the destination target, root by default
1210
1211         --limit     integer
1212                 The maximal sync speed in kBytes/s, default is unlimited
1213
1214         --maxsnap   integer
1215                 The number of snapshots to keep until older ones are erased.
1216                 The default is 1, use 0 for unlimited.
1217
1218         --dest-maxsnap   integer
1219                 Override maxsnap for the destination dataset.
1220
1221         --name      string
1222                 The name of the sync job, if not set it is 'default'.
1223                 It is only necessary if scheduler allready contains this source.
1224
1225         --prepend-storage-id
1226                 If specified, prepend the storage ID to the destination's path(s).
1227
1228         --source    string
1229                 The source can either be an <VMID> or [IP:]<ZFSPool>[/Path]
1230
1231         --source-user    string
1232                 The name of the user on the source target, root by default
1233
1234         --verbose
1235                 If specified, print out the sync progress.
1236
1237         --properties
1238                 If specified, include the dataset's properties in the stream.
1239
1240         --dest-config-path    string
1241                 Specifies a custom config path on the destination target.
1242                 The default is /var/lib/pve-zsync
1243     },
1244     list => qq{
1245 $PROGNAME list
1246
1247         Get a List of all scheduled Sync Jobs
1248     },
1249     status => qq{
1250 $PROGNAME status
1251
1252         Get the status of all scheduled Sync Jobs
1253     },
1254     help => qq{
1255 $PROGNAME help <cmd> [OPTIONS]
1256
1257     Get help about specified command.
1258
1259         <cmd>      string
1260                 Command name to get help about.
1261
1262         --verbose
1263                 Verbose output format.
1264     },
1265     enable => qq{
1266 $PROGNAME enable --source <string> [OPTIONS]
1267
1268     Enable a sync-job and reset all job-errors, if any.
1269
1270         --name      string
1271                 name of the sync job, if not set it is default
1272
1273         --source    string
1274                 the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1275     },
1276     disable => qq{
1277 $PROGNAME disable --source <string> [OPTIONS]
1278
1279     Disables (pauses) a sync-job
1280
1281         --name      string
1282                 name of the sync-job, if not set it is default
1283
1284         --source    string
1285                 the source can be an  <VMID> or [IP:]<ZFSPool>[/Path]
1286     },
1287     printpod => "$PROGNAME printpod\n\n\tinternal command",
1288
1289 };
1290
1291 if (!$command) {
1292     usage(); die "\n";
1293 } elsif (!$cmd_help->{$command}) {
1294     print "ERROR: unknown command '$command'";
1295     usage(1); die "\n";
1296 }
1297
1298 my @arg = @ARGV;
1299 my $param = parse_argv(@arg);
1300
1301 sub check_params {
1302     for (@_) {
1303         die "$cmd_help->{$command}\n" if !$param->{$_};
1304     }
1305 }
1306
1307 if ($command eq 'destroy') {
1308     check_params(qw(source));
1309
1310     check_target($param->{source});
1311     destroy_job($param);
1312
1313 } elsif ($command eq 'sync') {
1314     check_params(qw(source dest));
1315
1316     check_target($param->{source});
1317     check_target($param->{dest});
1318     sync($param);
1319
1320 } elsif ($command eq 'create') {
1321     check_params(qw(source dest));
1322
1323     check_target($param->{source});
1324     check_target($param->{dest});
1325     init($param);
1326
1327 } elsif ($command eq 'status') {
1328     print status();
1329
1330 } elsif ($command eq 'list') {
1331     print list();
1332
1333 } elsif ($command eq 'help') {
1334     my $help_command = $ARGV[1];
1335
1336     if ($help_command && $cmd_help->{$help_command}) {
1337         die "$cmd_help->{$help_command}\n";
1338
1339     }
1340     if ($param->{verbose}) {
1341         exec("man $PROGNAME");
1342
1343     } else {
1344         usage(1);
1345
1346     }
1347
1348 } elsif ($command eq 'enable') {
1349     check_params(qw(source));
1350
1351     check_target($param->{source});
1352     enable_job($param);
1353
1354 } elsif ($command eq 'disable') {
1355     check_params(qw(source));
1356
1357     check_target($param->{source});
1358     disable_job($param);
1359
1360 } elsif ($command eq 'printpod') {
1361     print_pod();
1362 }
1363
1364 sub usage {
1365     my ($help) = @_;
1366
1367     print("ERROR:\tno command specified\n") if !$help;
1368     print("USAGE:\t$PROGNAME <COMMAND> [ARGS] [OPTIONS]\n");
1369     print("\t$PROGNAME help [<cmd>] [OPTIONS]\n\n");
1370     print("\t$PROGNAME create --dest <string> --source <string> [OPTIONS]\n");
1371     print("\t$PROGNAME destroy --source <string> [OPTIONS]\n");
1372     print("\t$PROGNAME disable --source <string> [OPTIONS]\n");
1373     print("\t$PROGNAME enable --source <string> [OPTIONS]\n");
1374     print("\t$PROGNAME list\n");
1375     print("\t$PROGNAME status\n");
1376     print("\t$PROGNAME sync --dest <string> --source <string> [OPTIONS]\n");
1377 }
1378
1379 sub check_target {
1380     my ($target) = @_;
1381     parse_target($target);
1382 }
1383
1384 sub print_pod {
1385
1386     my $synopsis = join("\n", sort values %$cmd_help);
1387     my $commands = join(", ", sort keys %$cmd_help);
1388
1389     print <<EOF;
1390 =head1 NAME
1391
1392 pve-zsync - PVE ZFS Storage Sync Tool
1393
1394 =head1 SYNOPSIS
1395
1396 pve-zsync <COMMAND> [ARGS] [OPTIONS]
1397
1398 Where <COMMAND> can be one of: $commands
1399
1400 =head1 DESCRIPTION
1401
1402 The pve-zsync tool can help you to sync your VMs or directories stored on ZFS
1403 between multiple servers.
1404
1405 pve-zsync is able to automatically configure CRON jobs, so that a periodic sync
1406 will be automatically triggered.
1407 The default sync interval is 15 min, if you want to change this value you can
1408 do this in F</etc/cron.d/pve-zsync>. If you need help to configure CRON tabs, see
1409 man crontab.
1410
1411 =head1 COMMANDS AND OPTIONS
1412
1413 $synopsis
1414
1415 =head1 EXAMPLES
1416
1417 Adds a job for syncing the local VM 100 to a remote server's ZFS pool named "tank":
1418     pve-zsync create --source=100 -dest=192.168.1.2:tank
1419
1420 =head1 IMPORTANT FILES
1421
1422 Cron jobs and config are stored in F</etc/cron.d/pve-zsync>
1423
1424 The VM configuration itself gets copied to the destination machines
1425 F</var/lib/pve-zsync/> path.
1426
1427 =head1 COPYRIGHT AND DISCLAIMER
1428
1429 Copyright (C) 2007-2021 Proxmox Server Solutions GmbH
1430
1431 This program is free software: you can redistribute it and/or modify it under
1432 the terms of the GNU Affero General Public License as published by the Free
1433 Software Foundation, either version 3 of the License, or (at your option) any
1434 later version.
1435
1436 This program is distributed in the hope that it will be useful, but WITHOUT ANY
1437 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
1438 PARTICULAR PURPOSE. See the GNU Affero General Public License for more
1439 details.
1440
1441 You should have received a copy of the GNU Affero General Public License along
1442 with this program. If not, see <http://www.gnu.org/licenses/>.
1443
1444 EOF
1445 }