replicaiton: log rate and transport type

[pve-guest-common.git] / PVE / Replication.pm

diff --git a/PVE/Replication.pm b/PVE/Replication.pm
index 4d1e071e30bc075c4cbb2e2f2d8e5740580cf86b..b0c6db3beb967dd32beefb4a3b8674363e7f6d35 100644 (file)
--- a/PVE/Replication.pm
+++ b/PVE/Replication.pm
@@ -11,10 +11,12 @@ use PVE::INotify;
 use PVE::ProcFSTools;
 use PVE::Tools;
 use PVE::Cluster;
+use PVE::DataCenterConfig;
 use PVE::Storage;
 use PVE::GuestHelpers;
 use PVE::ReplicationConfig;
 use PVE::ReplicationState;
+use PVE::SSHInfo;
 
 
 # regression tests should overwrite this
@@ -54,6 +56,21 @@ sub find_common_replication_snapshot {
                     ($last_snapshots->{$volid}->{$parent_snapname} &&
                      $remote_snapshots->{$volid}->{$parent_snapname})) {
                $base_snapshots->{$volid} = $parent_snapname;
+           } elsif ($last_sync == 0) {
+               my @desc_sorted_snap =
+                   map { $_->[1] } sort { $b->[0] <=> $a->[0] }
+                   map { [ ($_ =~ /__replicate_\Q$jobid\E_(\d+)_/)[0] || 0, $_ ] }
+                   keys %{$remote_snapshots->{$volid}};
+
+               foreach my $remote_snap (@desc_sorted_snap) {
+                   if (defined($last_snapshots->{$volid}->{$remote_snap})) {
+                       $base_snapshots->{$volid} = $remote_snap;
+                       last;
+                   }
+               }
+               die "No common base to restore the job state\n".
+                   "please delete jobid: $jobid and create the job again\n"
+                   if !defined($base_snapshots->{$volid});
            }
        }
     }
@@ -64,7 +81,7 @@ sub find_common_replication_snapshot {
 sub remote_prepare_local_job {
     my ($ssh_info, $jobid, $vmid, $volumes, $storeid_list, $last_sync, $parent_snapname, $force, $logfunc) = @_;
 
-    my $ssh_cmd = PVE::Cluster::ssh_info_to_command($ssh_info);
+    my $ssh_cmd = PVE::SSHInfo::ssh_info_to_command($ssh_info);
     my $cmd = [@$ssh_cmd, '--', 'pvesr', 'prepare-local-job', $jobid];
     push @$cmd, '--scan', join(',', @$storeid_list) if scalar(@$storeid_list);
     push @$cmd, @$volumes if scalar(@$volumes);
@@ -98,7 +115,7 @@ sub remote_prepare_local_job {
 sub remote_finalize_local_job {
     my ($ssh_info, $jobid, $vmid, $volumes, $last_sync, $logfunc) = @_;
 
-    my $ssh_cmd = PVE::Cluster::ssh_info_to_command($ssh_info);
+    my $ssh_cmd = PVE::SSHInfo::ssh_info_to_command($ssh_info);
     my $cmd = [@$ssh_cmd, '--', 'pvesr', 'finalize-local-job', $jobid,
               @$volumes, '--last_sync', $last_sync];
 
@@ -135,22 +152,26 @@ sub prepare {
                (defined($parent_snapname) && ($snap eq $parent_snapname))) {
                $last_snapshots->{$volid}->{$snap} = 1;
            } elsif ($snap =~ m/^\Q$prefix\E/) {
-               $logfunc->("delete stale replication snapshot '$snap' on $volid");
-
-               eval {
-                   PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap);
-                   $cleaned_replicated_volumes->{$volid} = 1;
-               };
-
-               # If deleting the snapshot fails, we can not be sure if it was due to an error or a timeout.
-               # The likelihood that the delete has worked out is high at a timeout.
-               # If it really fails, it will try to remove on the next run.
-               if (my $err = $@) {
-                   # warn is for syslog/journal.
-                   warn $err;
-
-                   # logfunc will written in replication log.
-                   $logfunc->("delete stale replication snapshot error: $err");
+               if ($last_sync != 0) {
+                   $logfunc->("delete stale replication snapshot '$snap' on $volid");
+                   eval {
+                       PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap);
+                       $cleaned_replicated_volumes->{$volid} = 1;
+                   };
+
+                   # If deleting the snapshot fails, we can not be sure if it was due to an error or a timeout.
+                   # The likelihood that the delete has worked out is high at a timeout.
+                   # If it really fails, it will try to remove on the next run.
+                   if (my $err = $@) {
+                       # warn is for syslog/journal.
+                       warn $err;
+
+                       # logfunc will written in replication log.
+                       $logfunc->("delete stale replication snapshot error: $err");
+                   }           
+               # Last_sync=0 and a replication snapshot only occur, if the VM was stolen
+               } else {
+                   $last_snapshots->{$volid}->{$snap} = 1;
                }
            }
        }
@@ -164,7 +185,8 @@ sub replicate_volume {
 
     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 
-    my $ratelimit_bps = int(1000000*$rate) if $rate;
+    my $ratelimit_bps = $rate ? int(1000000 * $rate) : undef;
+
     PVE::Storage::storage_migrate($storecfg, $volid, $ssh_info, $storeid, $volname,
                                  $base_snapshot, $sync_snapname, $ratelimit_bps, $insecure, 1, $logfunc);
 }
@@ -218,13 +240,12 @@ sub replicate {
 
            my %hash = map { $_ => 1 } @store_list;
 
-           my $ssh_info = PVE::Cluster::get_ssh_info($jobcfg->{target});
-
+           my $ssh_info = PVE::SSHInfo::get_ssh_info($jobcfg->{target});
            remote_prepare_local_job($ssh_info, $jobid, $vmid, [], [ keys %hash ], 1, undef, 1, $logfunc);
 
        }
        # remove all local replication snapshots (lastsync => 0)
-       prepare($storecfg, $sorted_volids, $jobid, 0, undef, $logfunc);
+       prepare($storecfg, $sorted_volids, $jobid, 1, undef, $logfunc);
 
        PVE::ReplicationConfig::delete_job($jobid); # update config
        $logfunc->("job removed");
@@ -232,7 +253,7 @@ sub replicate {
        return undef;
     }
 
-    my $ssh_info = PVE::Cluster::get_ssh_info($jobcfg->{target}, $migration_network);
+    my $ssh_info = PVE::SSHInfo::get_ssh_info($jobcfg->{target}, $migration_network);
 
     my $parent_snapname = $conf->{parent};
 
@@ -291,6 +312,9 @@ sub replicate {
        my $rate = $jobcfg->{rate};
        my $insecure = $migration_type eq 'insecure';
 
+       $logfunc->("using $migration_type transmission, rate limit: "
+           . ($rate ? "$rate MByte/s" : "none"));
+
        foreach my $volid (@$sorted_volids) {
            my $base_snapname;
 
@@ -337,7 +361,7 @@ my $run_replication_nolock = sub {
 
     my $volumes;
 
-    # we normaly write errors into the state file,
+    # we normally write errors into the state file,
     # but we also catch unexpected errors and log them to syslog
     # (for examply when there are problems writing the state file)