]>
Commit | Line | Data |
---|---|---|
892821fd DM |
1 | package PVE::CLI::pvesr; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | use POSIX qw(strftime); | |
6 | use JSON; | |
7 | ||
8 | use PVE::JSONSchema qw(get_standard_option); | |
9 | use PVE::INotify; | |
10 | use PVE::RPCEnvironment; | |
11 | use PVE::Tools qw(extract_param); | |
12 | use PVE::SafeSyslog; | |
13 | use PVE::CLIHandler; | |
14 | ||
fae99506 | 15 | use PVE::Cluster; |
892821fd DM |
16 | use PVE::Replication; |
17 | use PVE::API2::ReplicationConfig; | |
18 | use PVE::API2::Replication; | |
19 | ||
20 | use base qw(PVE::CLIHandler); | |
21 | ||
22 | my $nodename = PVE::INotify::nodename(); | |
23 | ||
24 | sub setup_environment { | |
25 | PVE::RPCEnvironment->setup_default_cli_env(); | |
26 | } | |
27 | ||
91ee6a2f DM |
28 | # fixme: get from plugin?? |
29 | my $replicatable_storage_types = { | |
30 | zfspool => 1, | |
31 | }; | |
32 | ||
33 | my $check_wanted_volid = sub { | |
34 | my ($storecfg, $vmid, $volid, $local_node) = @_; | |
35 | ||
36 | my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); | |
37 | my $scfg = PVE::Storage::storage_check_enabled($storecfg, $storeid, $local_node); | |
38 | die "storage '$storeid' is not replicatable\n" | |
39 | if !$replicatable_storage_types->{$scfg->{type}}; | |
40 | ||
41 | my ($vtype, undef, $ownervm) = PVE::Storage::parse_volname($storecfg, $volid); | |
42 | die "volume '$volid' has wrong vtype ($vtype != 'images')\n" | |
43 | if $vtype ne 'images'; | |
44 | die "volume '$volid' has wrong owner\n" | |
45 | if !$ownervm || $vmid != $ownervm; | |
46 | ||
47 | return $storeid; | |
48 | }; | |
49 | ||
fae99506 DM |
50 | __PACKAGE__->register_method ({ |
51 | name => 'prepare_local_job', | |
52 | path => 'prepare_local_job', | |
53 | method => 'POST', | |
54 | description => "Prepare for starting a replication job. This is called on the target node before replication starts. This call is for internal use, and return a JSON object on stdout. The method first test if VM <vmid> reside on the local node. If so, stop immediately. After that the method scans all volume IDs for snapshots, and removes all replications snapshots with timestamps different than <last_sync>. It also removes any unused volumes. Returns a hash with boolean markers for all volumes with existing replication snapshots.", | |
55 | parameters => { | |
56 | additionalProperties => 0, | |
57 | properties => { | |
58 | id => get_standard_option('pve-replication-id'), | |
fae99506 DM |
59 | 'extra-args' => get_standard_option('extra-args', { |
60 | description => "The list of volume IDs to consider." }), | |
91ee6a2f DM |
61 | scan => { |
62 | description => "List of storage IDs to scan for stale volumes.", | |
63 | type => 'string', format => 'pve-storage-id-list', | |
64 | optional => 1, | |
65 | }, | |
f9d38c54 DM |
66 | force => { |
67 | description => "Allow to remove all existion volumes (empty volume list).", | |
68 | type => 'boolean', | |
69 | optional => 1, | |
70 | default => 0, | |
71 | }, | |
fae99506 DM |
72 | last_sync => { |
73 | description => "Time (UNIX epoch) of last successful sync. If not specified, all replication snapshots get removed.", | |
74 | type => 'integer', | |
75 | minimum => 0, | |
76 | optional => 1, | |
77 | }, | |
91ee6a2f DM |
78 | parent_snapname => get_standard_option('pve-snapshot-name', { |
79 | optional => 1, | |
80 | }), | |
fae99506 DM |
81 | }, |
82 | }, | |
83 | returns => { type => 'null' }, | |
84 | code => sub { | |
85 | my ($param) = @_; | |
86 | ||
91ee6a2f DM |
87 | my $logfunc = sub { |
88 | my ($msg) = @_; | |
89 | print STDERR "$msg\n"; | |
90 | }; | |
fae99506 DM |
91 | |
92 | my $local_node = PVE::INotify::nodename(); | |
93 | ||
91ee6a2f DM |
94 | die "no volumes specified\n" |
95 | if !$param->{force} && !scalar(@{$param->{'extra-args'}}); | |
96 | ||
97 | my ($vmid, undef, $jobid) = PVE::ReplicationConfig::parse_replication_job_id($param->{id}); | |
98 | ||
fae99506 DM |
99 | my $vms = PVE::Cluster::get_vmlist(); |
100 | die "guest '$vmid' is on local node\n" | |
101 | if $vms->{ids}->{$vmid} && $vms->{ids}->{$vmid}->{node} eq $local_node; | |
102 | ||
91ee6a2f DM |
103 | my $last_sync = $param->{last_sync} // 0; |
104 | my $parent_snapname = $param->{parent_snapname}; | |
fae99506 | 105 | |
91ee6a2f | 106 | my $storecfg = PVE::Storage::config(); |
fae99506 | 107 | |
91ee6a2f DM |
108 | # compute list of storages we want to scan |
109 | my $storage_hash = {}; | |
110 | foreach my $storeid (PVE::Tools::split_list($param->{scan})) { | |
111 | my $scfg = PVE::Storage::storage_check_enabled($storecfg, $storeid, $local_node, 1); | |
112 | next if !$scfg; # simply ignore unavailable storages here | |
113 | die "storage '$storeid' is not replicatable\n" if !$replicatable_storage_types->{$scfg->{type}}; | |
114 | $storage_hash->{$storeid} = 1; | |
115 | } | |
fae99506 | 116 | |
91ee6a2f | 117 | my $wanted_volids = {}; |
fae99506 | 118 | foreach my $volid (@{$param->{'extra-args'}}) { |
91ee6a2f DM |
119 | my $storeid = $check_wanted_volid->($storecfg, $vmid, $volid, $local_node); |
120 | $wanted_volids->{$volid} = 1; | |
121 | $storage_hash->{$storeid} = 1; | |
fae99506 | 122 | } |
91ee6a2f DM |
123 | my $storage_list = [ sort keys %$storage_hash ]; |
124 | ||
125 | # activate all used storage | |
126 | my $cache = {}; | |
127 | PVE::Storage::activate_storage_list($storecfg, $storage_list, $cache); | |
128 | ||
129 | my $snapname = PVE::ReplicationState::replication_snapshot_name($jobid, $last_sync); | |
130 | ||
131 | # find replication snapshots | |
544bb4e2 | 132 | my $volids = []; |
91ee6a2f DM |
133 | foreach my $storeid (@$storage_list) { |
134 | my $scfg = PVE::Storage::storage_config($storecfg, $storeid); | |
135 | my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); | |
37efc934 | 136 | my $images = $plugin->list_images($storeid, $scfg, $vmid, undef, $cache); |
544bb4e2 FG |
137 | push @$volids, map { $_->{volid} } @$images; |
138 | } | |
e3e0e096 | 139 | my ($local_snapshots, $cleaned_replicated_volumes) = PVE::Replication::prepare($storecfg, $volids, $jobid, $last_sync, $parent_snapname, $logfunc); |
544bb4e2 FG |
140 | foreach my $volid (keys %$cleaned_replicated_volumes) { |
141 | if (!$wanted_volids->{$volid}) { | |
142 | $logfunc->("$jobid: delete stale volume '$volid'"); | |
143 | PVE::Storage::vdisk_free($storecfg, $volid); | |
e3e0e096 | 144 | delete $local_snapshots->{$volid}; |
fae99506 DM |
145 | } |
146 | } | |
147 | ||
e3e0e096 | 148 | print to_json($local_snapshots) . "\n"; |
fae99506 DM |
149 | |
150 | return undef; | |
151 | }}); | |
152 | ||
fcc22c0b DM |
153 | __PACKAGE__->register_method ({ |
154 | name => 'finalize_local_job', | |
155 | path => 'finalize_local_job', | |
156 | method => 'POST', | |
157 | description => "Finalize a replication job. This removes all replications snapshots with timestamps different than <last_sync>.", | |
158 | parameters => { | |
159 | additionalProperties => 0, | |
160 | properties => { | |
161 | id => get_standard_option('pve-replication-id'), | |
fcc22c0b DM |
162 | 'extra-args' => get_standard_option('extra-args', { |
163 | description => "The list of volume IDs to consider." }), | |
164 | last_sync => { | |
165 | description => "Time (UNIX epoch) of last successful sync. If not specified, all replication snapshots gets removed.", | |
166 | type => 'integer', | |
167 | minimum => 0, | |
168 | optional => 1, | |
169 | }, | |
170 | }, | |
171 | }, | |
172 | returns => { type => 'null' }, | |
173 | code => sub { | |
174 | my ($param) = @_; | |
175 | ||
a9da300d | 176 | my ($vmid, undef, $jobid) = PVE::ReplicationConfig::parse_replication_job_id($param->{id}); |
fcc22c0b DM |
177 | my $last_sync = $param->{last_sync} // 0; |
178 | ||
179 | my $local_node = PVE::INotify::nodename(); | |
180 | ||
181 | my $vms = PVE::Cluster::get_vmlist(); | |
182 | die "guest '$vmid' is on local node\n" | |
183 | if $vms->{ids}->{$vmid} && $vms->{ids}->{$vmid}->{node} eq $local_node; | |
184 | ||
185 | my $storecfg = PVE::Storage::config(); | |
186 | ||
187 | my $volids = []; | |
188 | ||
189 | die "no volumes specified\n" if !scalar(@{$param->{'extra-args'}}); | |
190 | ||
191 | foreach my $volid (@{$param->{'extra-args'}}) { | |
91ee6a2f | 192 | $check_wanted_volid->($storecfg, $vmid, $volid, $local_node); |
fcc22c0b DM |
193 | push @$volids, $volid; |
194 | } | |
195 | ||
196 | $volids = [ sort @$volids ]; | |
197 | ||
198 | my $logfunc = sub { | |
c364b61f | 199 | my ($msg) = @_; |
fcc22c0b DM |
200 | print STDERR "$msg\n"; |
201 | }; | |
202 | ||
e3e0e096 | 203 | PVE::Replication::prepare($storecfg, $volids, $jobid, $last_sync, undef, $logfunc); |
fcc22c0b DM |
204 | |
205 | return undef; | |
206 | }}); | |
207 | ||
892821fd DM |
208 | __PACKAGE__->register_method ({ |
209 | name => 'run', | |
210 | path => 'run', | |
211 | method => 'POST', | |
212 | description => "This method is called by the systemd-timer and executes all (or a specific) sync jobs.", | |
213 | parameters => { | |
214 | additionalProperties => 0, | |
215 | properties => { | |
216 | id => get_standard_option('pve-replication-id', { optional => 1 }), | |
f70997ea DM |
217 | verbose => { |
218 | description => "Print more verbose logs to stdout.", | |
219 | type => 'boolean', | |
220 | default => 0, | |
221 | optional => 1, | |
222 | }, | |
64d39c2e WL |
223 | mail => { |
224 | description => "Send an email notification in case of a failure.", | |
225 | type => 'boolean', | |
226 | default => 0, | |
227 | optional => 1, | |
228 | }, | |
892821fd DM |
229 | }, |
230 | }, | |
231 | returns => { type => 'null' }, | |
232 | code => sub { | |
233 | my ($param) = @_; | |
234 | ||
64d39c2e WL |
235 | die "Mail and id are mutually exclusive!\n" |
236 | if $param->{id} && $param->{mail}; | |
237 | ||
f70997ea DM |
238 | my $logfunc; |
239 | ||
240 | if ($param->{verbose}) { | |
241 | $logfunc = sub { | |
c364b61f | 242 | my ($msg) = @_; |
f70997ea DM |
243 | print "$msg\n"; |
244 | }; | |
245 | } | |
246 | ||
892821fd DM |
247 | if (my $id = extract_param($param, 'id')) { |
248 | ||
810c6776 | 249 | PVE::API2::Replication::run_single_job($id, undef, $logfunc); |
892821fd DM |
250 | |
251 | } else { | |
252 | ||
64d39c2e | 253 | PVE::API2::Replication::run_jobs(undef, $logfunc, 0, $param->{mail}); |
892821fd DM |
254 | } |
255 | ||
256 | return undef; | |
257 | }}); | |
258 | ||
259 | __PACKAGE__->register_method ({ | |
260 | name => 'enable', | |
261 | path => 'enable', | |
262 | method => 'POST', | |
263 | description => "Enable a replication job.", | |
264 | parameters => { | |
265 | additionalProperties => 0, | |
266 | properties => { | |
267 | id => get_standard_option('pve-replication-id'), | |
268 | }, | |
269 | }, | |
270 | returns => { type => 'null' }, | |
271 | code => sub { | |
272 | my ($param) = @_; | |
273 | ||
274 | $param->{disable} = 0; | |
275 | ||
276 | return PVE::API2::ReplicationConfig->update($param); | |
277 | }}); | |
278 | ||
279 | __PACKAGE__->register_method ({ | |
280 | name => 'disable', | |
281 | path => 'disable', | |
282 | method => 'POST', | |
283 | description => "Disable a replication job.", | |
284 | parameters => { | |
285 | additionalProperties => 0, | |
286 | properties => { | |
287 | id => get_standard_option('pve-replication-id'), | |
288 | }, | |
289 | }, | |
290 | returns => { type => 'null' }, | |
291 | code => sub { | |
292 | my ($param) = @_; | |
293 | ||
294 | $param->{disable} = 1; | |
295 | ||
296 | return PVE::API2::ReplicationConfig->update($param); | |
297 | }}); | |
298 | ||
fdc8b769 WL |
299 | __PACKAGE__->register_method ({ |
300 | name => 'set_state', | |
301 | path => '', | |
302 | protected => 1, | |
303 | method => 'POST', | |
304 | description => "Set the job replication state on migration. This call is for internal use. It will accept the job state as ja JSON obj.", | |
305 | permissions => { | |
306 | check => ['perm', '/storage', ['Datastore.Allocate']], | |
307 | }, | |
308 | parameters => { | |
309 | additionalProperties => 0, | |
310 | properties => { | |
311 | vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_vmid }), | |
312 | state => { | |
313 | description => "Job state as JSON decoded string.", | |
314 | type => 'string', | |
315 | }, | |
316 | }, | |
317 | }, | |
318 | returns => { type => 'null' }, | |
319 | code => sub { | |
320 | my ($param) = @_; | |
321 | ||
322 | my $vmid = extract_param($param, 'vmid'); | |
323 | my $json_string = extract_param($param, 'state'); | |
324 | my $remote_job_state= decode_json $json_string; | |
325 | ||
326 | PVE::ReplicationState::write_vmid_job_states($remote_job_state, $vmid); | |
327 | return undef; | |
328 | }}); | |
329 | ||
892821fd DM |
330 | my $print_job_list = sub { |
331 | my ($list) = @_; | |
332 | ||
a9da300d | 333 | my $format = "%-20s %-20s %10s %5s %8s\n"; |
892821fd | 334 | |
a9da300d | 335 | printf($format, "JobID", "Target", "Schedule", "Rate", "Enabled"); |
892821fd DM |
336 | |
337 | foreach my $job (sort { $a->{guest} <=> $b->{guest} } @$list) { | |
338 | my $plugin = PVE::ReplicationConfig->lookup($job->{type}); | |
339 | my $tid = $plugin->get_unique_target_id($job); | |
340 | ||
a9da300d | 341 | printf($format, $job->{id}, $tid, |
5c180db3 | 342 | defined($job->{schedule}) ? $job->{schedule} : '*/15', |
892821fd DM |
343 | defined($job->{rate}) ? $job->{rate} : '-', |
344 | $job->{disable} ? 'no' : 'yes' | |
345 | ); | |
346 | } | |
347 | }; | |
348 | ||
349 | my $print_job_status = sub { | |
350 | my ($list) = @_; | |
351 | ||
0cfec28c | 352 | my $format = "%-10s %-10s %-20s %20s %20s %10s %10s %s\n"; |
892821fd | 353 | |
0cfec28c | 354 | printf($format, "JobID", "Enabled", "Target", "LastSync", "NextSync", "Duration", "FailCount", "State"); |
892821fd DM |
355 | |
356 | foreach my $job (sort { $a->{guest} <=> $b->{guest} } @$list) { | |
357 | my $plugin = PVE::ReplicationConfig->lookup($job->{type}); | |
358 | my $tid = $plugin->get_unique_target_id($job); | |
359 | ||
5c180db3 DM |
360 | my $timestr = '-'; |
361 | if ($job->{last_sync}) { | |
362 | $timestr = strftime("%Y-%m-%d_%H:%M:%S", localtime($job->{last_sync})); | |
363 | } | |
364 | ||
365 | my $nextstr = '-'; | |
366 | if (my $next = $job->{next_sync}) { | |
367 | my $now = time(); | |
368 | if ($next > $now) { | |
369 | $nextstr = strftime("%Y-%m-%d_%H:%M:%S", localtime($job->{next_sync})); | |
370 | } else { | |
0b142737 | 371 | $nextstr = 'pending'; |
5c180db3 DM |
372 | } |
373 | } | |
892821fd | 374 | |
483f89dd | 375 | my $state = $job->{pid} ? "SYNCING" : $job->{error} // 'OK'; |
0cfec28c | 376 | my $enabled = $job->{disable} ? 'No' : 'Yes'; |
483f89dd | 377 | |
0cfec28c | 378 | printf($format, $job->{id}, $enabled, $tid, |
5c180db3 | 379 | $timestr, $nextstr, $job->{duration} // '-', |
483f89dd | 380 | $job->{fail_count}, $state); |
892821fd DM |
381 | } |
382 | }; | |
383 | ||
384 | our $cmddef = { | |
385 | status => [ 'PVE::API2::Replication', 'status', [], { node => $nodename }, $print_job_status ], | |
ca7d0707 | 386 | 'schedule-now' => [ 'PVE::API2::Replication', 'schedule_now', ['id'], { node => $nodename }], |
892821fd | 387 | |
5c180db3 | 388 | list => [ 'PVE::API2::ReplicationConfig', 'index' , [], {}, $print_job_list ], |
892821fd DM |
389 | read => [ 'PVE::API2::ReplicationConfig', 'read' , ['id'], {}, |
390 | sub { my $res = shift; print to_json($res, { utf8 => 1, pretty => 1, canonical => 1}); }], | |
391 | update => [ 'PVE::API2::ReplicationConfig', 'update' , ['id'], {} ], | |
392 | delete => [ 'PVE::API2::ReplicationConfig', 'delete' , ['id'], {} ], | |
a9da300d | 393 | 'create-local-job' => [ 'PVE::API2::ReplicationConfig', 'create' , ['id', 'target'], |
892821fd DM |
394 | { type => 'local' } ], |
395 | ||
396 | enable => [ __PACKAGE__, 'enable', ['id'], {}], | |
397 | disable => [ __PACKAGE__, 'disable', ['id'], {}], | |
398 | ||
a9da300d DM |
399 | 'prepare-local-job' => [ __PACKAGE__, 'prepare_local_job', ['id', 'extra-args'], {} ], |
400 | 'finalize-local-job' => [ __PACKAGE__, 'finalize_local_job', ['id', 'extra-args'], {} ], | |
fae99506 | 401 | |
892821fd | 402 | run => [ __PACKAGE__ , 'run'], |
fdc8b769 | 403 | 'set-state' => [ __PACKAGE__ , 'set_state', ['vmid', 'state']], |
892821fd DM |
404 | }; |
405 | ||
406 | 1; |