PVE::Replication - use new calendar events instead of interval

And implement retry algorythm after failure:

  $next_sync = $state->{last_try} + 5*60*$fail_count;

and limit to 3 failures.

diff --git a/PVE/API2/Replication.pm b/PVE/API2/Replication.pm
index d1d8feb236efde8efcdcd9bdf7c805c66dc57bfe..192a61bbf605da1bf6d0f7f1b249d224a69d12df 100644 (file)
--- a/PVE/API2/Replication.pm
+++ b/PVE/API2/Replication.pm
@@ -82,7 +82,7 @@ __PACKAGE__->register_method ({
            my $vmid = $d->{guest};
            next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ]);
            $d->{id} = $id;
-           foreach my $k (qw(last_sync fail_count error duration)) {
+           foreach my $k (qw(last_sync last_try fail_count error duration)) {
                $d->{$k} = $state->{$k} if defined($state->{$k});
            }
            if ($state->{pid} && $state->{ptime}) {

diff --git a/PVE/CLI/pvesr.pm b/PVE/CLI/pvesr.pm
index 49fc86a3bc7ee91b1e1b2b31bcdd4aa75a57d5e2..be5ae5eecd48e53e4cfcd7a5b0e363bab802a49c 100644 (file)
--- a/PVE/CLI/pvesr.pm
+++ b/PVE/CLI/pvesr.pm
@@ -96,14 +96,14 @@ my $print_job_list = sub {
 
     my $format = "%-20s %10s %-20s %10s %5s %8s\n";
 
-    printf($format, "JobID", "GuestID", "Target", "Interval", "Rate", "Enabled");
+    printf($format, "JobID", "GuestID", "Target", "Schedule", "Rate", "Enabled");
 
     foreach my $job (sort { $a->{guest} <=> $b->{guest} } @$list) {
        my $plugin = PVE::ReplicationConfig->lookup($job->{type});
        my $tid = $plugin->get_unique_target_id($job);
 
        printf($format, $job->{id}, $job->{guest}, $tid,
-              defined($job->{interval}) ? $job->{interval} : '-',
+              defined($job->{schedule}) ? $job->{schedule} : '*/15',
               defined($job->{rate}) ? $job->{rate} : '-',
               $job->{disable} ? 'no' : 'yes'
            );
@@ -113,21 +113,33 @@ my $print_job_list = sub {
 my $print_job_status = sub {
     my ($list) = @_;
 
-    my $format = "%-20s %10s %-20s %20s %10s %10s %s\n";
+    my $format = "%-20s %10s %-20s %20s %20s %10s %10s %s\n";
 
-    printf($format, "JobID", "GuestID", "Target", "LastSync", "Duration", "FailCount", "State");
+    printf($format, "JobID", "GuestID", "Target", "LastSync", "NextSync", "Duration", "FailCount", "State");
 
     foreach my $job (sort { $a->{guest} <=> $b->{guest} } @$list) {
        my $plugin = PVE::ReplicationConfig->lookup($job->{type});
        my $tid = $plugin->get_unique_target_id($job);
 
-       my $timestr = $job->{last_sync} ?
-           strftime("%Y-%m-%d_%H:%M:%S", localtime($job->{last_sync})) : '-';
+       my $timestr = '-';
+       if ($job->{last_sync}) {
+           $timestr = strftime("%Y-%m-%d_%H:%M:%S", localtime($job->{last_sync}));
+       }
+
+       my $nextstr = '-';
+       if (my $next = $job->{next_sync}) {
+           my $now = time();
+           if ($next > $now) {
+               $nextstr = strftime("%Y-%m-%d_%H:%M:%S", localtime($job->{next_sync}));
+           } else {
+               $nextstr = 'now';
+           }
+       }
 
        my $state = $job->{pid} ? "SYNCING" : $job->{error} // 'OK';
 
        printf($format, $job->{id}, $job->{guest}, $tid,
-              $timestr, $job->{duration} // '-',
+              $timestr, $nextstr, $job->{duration} // '-',
               $job->{fail_count}, $state);
     }
 };
@@ -135,7 +147,7 @@ my $print_job_status = sub {
 our $cmddef = {
     status => [ 'PVE::API2::Replication', 'status', [], { node => $nodename }, $print_job_status ],
 
-    jobs => [ 'PVE::API2::ReplicationConfig', 'index' , [], {}, $print_job_list ],
+    list => [ 'PVE::API2::ReplicationConfig', 'index' , [], {}, $print_job_list ],
     read => [ 'PVE::API2::ReplicationConfig', 'read' , ['id'], {},
             sub { my $res = shift; print to_json($res, { utf8 => 1, pretty => 1, canonical => 1}); }],
     update => [ 'PVE::API2::ReplicationConfig', 'update' , ['id'], {} ],

diff --git a/PVE/Replication.pm b/PVE/Replication.pm
index 86e7dd5dac6081efa50e0dc681e9e0bba8fd9698..ab4f9ecaf0a88a7b92f33d694740612489b54847 100644 (file)
--- a/PVE/Replication.pm
+++ b/PVE/Replication.pm
@@ -9,6 +9,7 @@ use Time::HiRes qw(gettimeofday tv_interval);
 use PVE::INotify;
 use PVE::ProcFSTools;
 use PVE::Tools;
+use PVE::CalendarEvent;
 use PVE::Cluster;
 use PVE::QemuConfig;
 use PVE::QemuServer;
@@ -46,7 +47,8 @@ my $get_job_state = sub {
     $state = {} if !$state;
 
     $state->{last_iteration} //= 0;
-    $state->{last_sync} //= 0;
+    $state->{last_try} //= 0; # last sync start time
+    $state->{last_sync} //= 0; # last successful sync start time
     $state->{fail_count} //= 0;
 
     return $state;
@@ -93,10 +95,23 @@ sub job_status {
 
        next if $jobcfg->{disable};
 
-       $jobcfg->{state} = $get_job_state->($stateobj, $jobcfg);
+       my $state = $get_job_state->($stateobj, $jobcfg);
+       $jobcfg->{state} = $state;
        $jobcfg->{id} = $jobid;
        $jobcfg->{vmtype} = $vms->{ids}->{$vmid}->{type};
 
+       my $next_sync = 0;
+       if (my $fail_count = $state->{fail_count}) {
+           if ($fail_count < 3) {
+               $next_sync = $state->{last_try} + 5*60*$fail_count;
+           }
+       } else {
+           my $schedule =  $jobcfg->{schedule} || '*/15';
+           my $calspec = PVE::CalendarEvent::parse_calendar_event($schedule);
+           $next_sync = PVE::CalendarEvent::compute_next_event($calspec, $state->{last_try}) // 0;
+       }
+       $jobcfg->{next_sync} = $next_sync;
+
        $jobs->{$jobid} = $jobcfg;
     }
 
@@ -110,15 +125,6 @@ my $get_next_job = sub {
 
     my $jobs = job_status($stateobj);
 
-    # compute next_sync here to make it easy to sort jobs
-    my $next_sync_hash = {};
-    foreach my $jobid (keys %$jobs) {
-       my $jobcfg = $jobs->{$jobid};
-       my $interval = $jobcfg->{interval} || 15;
-       my $last_sync = $jobcfg->{state}->{last_sync};
-       $next_sync_hash->{$jobid} = $last_sync + $interval * 60;
-    }
-
     my $sort_func = sub {
        my $joba = $jobs->{$a};
        my $jobb = $jobs->{$b};
@@ -126,20 +132,15 @@ my $get_next_job = sub {
        my $sb =  $jobb->{state};
        my $res = $sa->{last_iteration} cmp $sb->{last_iteration};
        return $res if $res != 0;
-       $res = $next_sync_hash->{$a} <=> $next_sync_hash->{$b};
+       $res = $joba->{next_sync} <=> $jobb->{next_sync};
        return $res if $res != 0;
        return  $joba->{guest} <=> $jobb->{guest};
     };
 
     foreach my $jobid (sort $sort_func keys %$jobs) {
        my $jobcfg = $jobs->{$jobid};
-<<<<<<< HEAD
-       next if $jobcfg->{state}->{last_iteration} >= $now;
-       if ($now >= $next_sync_hash->{$jobid}) {
-=======
        next if $jobcfg->{state}->{last_iteration} >= $iteration;
        if ($jobcfg->{next_sync} && ($start_time >= $jobcfg->{next_sync})) {
->>>>>>> c2eac19e... fixup iteration marker
            $next_jobid = $jobid;
            last;
        }
@@ -179,7 +180,7 @@ my $run_replication = sub {
 
     $state->{pid} = $$;
     $state->{ptime} = PVE::ProcFSTools::read_proc_starttime($state->{pid});
-
+    $state->{last_try} = $start_time;
     $update_job_state->($stateobj, $jobcfg,  $state);
 
     eval { replicate($jobcfg, $start_time); };
@@ -193,6 +194,7 @@ my $run_replication = sub {
        $state->{fail_count}++;
        $state->{error} = "$err";
        $update_job_state->($stateobj, $jobcfg,  $state);
+       warn $err;
    } else {
        $state->{last_sync} = $start_time;
        $state->{fail_count} = 0;