]> git.proxmox.com Git - qemu-server.git/blame - PVE/QemuMigrate.pm
Update volume IDs in one go
[qemu-server.git] / PVE / QemuMigrate.pm
CommitLineData
3ea94c60 1package PVE::QemuMigrate;
1ef75254 2
1e3baf05 3use strict;
3ea94c60 4use warnings;
6d7450cb 5
3ea94c60 6use IO::File;
1e3baf05 7use IPC::Open2;
61b04c6d 8use POSIX qw( WNOHANG );
6d7450cb
TL
9use Time::HiRes qw( usleep );
10
3ea94c60 11use PVE::Cluster;
6d7450cb
TL
12use PVE::INotify;
13use PVE::RPCEnvironment;
14use PVE::Replication;
15use PVE::ReplicationConfig;
16use PVE::ReplicationState;
1e3baf05 17use PVE::Storage;
6d7450cb
TL
18use PVE::Tools;
19
912792e2 20use PVE::QemuConfig;
58c64ad5 21use PVE::QemuServer::CPUConfig;
e0fd2b2f 22use PVE::QemuServer::Drive;
28e6e180 23use PVE::QemuServer::Helpers qw(min_version);
3392d6ca 24use PVE::QemuServer::Machine;
0a13e08e 25use PVE::QemuServer::Monitor qw(mon_cmd);
6d7450cb 26use PVE::QemuServer;
1e3baf05 27
6d7450cb 28use PVE::AbstractMigrate;
16e903f2 29use base qw(PVE::AbstractMigrate);
1e3baf05 30
1ef75254 31sub fork_command_pipe {
46a84fd4 32 my ($self, $cmd) = @_;
19672434 33
1ef75254
DM
34 my $reader = IO::File->new();
35 my $writer = IO::File->new();
36
37 my $orig_pid = $$;
38
39 my $cpid;
40
41 eval { $cpid = open2($reader, $writer, @$cmd); };
42
43 my $err = $@;
44
45 # catch exec errors
46 if ($orig_pid != $$) {
46a84fd4 47 $self->log('err', "can't fork command pipe\n");
19672434
DM
48 POSIX::_exit(1);
49 kill('KILL', $$);
1ef75254
DM
50 }
51
52 die $err if $err;
53
54 return { writer => $writer, reader => $reader, pid => $cpid };
55}
56
19672434 57sub finish_command_pipe {
97439670 58 my ($self, $cmdpipe, $timeout) = @_;
1ef75254 59
61b04c6d
TL
60 my $cpid = $cmdpipe->{pid};
61 return if !defined($cpid);
62
1ef75254
DM
63 my $writer = $cmdpipe->{writer};
64 my $reader = $cmdpipe->{reader};
65
66 $writer->close();
67 $reader->close();
68
61b04c6d
TL
69 my $collect_child_process = sub {
70 my $res = waitpid($cpid, WNOHANG);
71 if (defined($res) && ($res == $cpid)) {
72 delete $cmdpipe->{cpid};
73 return 1;
74 } else {
75 return 0;
76 }
77 };
1ef75254 78
97439670
DM
79 if ($timeout) {
80 for (my $i = 0; $i < $timeout; $i++) {
61b04c6d 81 return if &$collect_child_process();
97439670
DM
82 sleep(1);
83 }
84 }
85
86 $self->log('info', "ssh tunnel still running - terminating now with SIGTERM\n");
87 kill(15, $cpid);
1ef75254 88
97439670
DM
89 # wait again
90 for (my $i = 0; $i < 10; $i++) {
61b04c6d 91 return if &$collect_child_process();
97439670
DM
92 sleep(1);
93 }
94
95 $self->log('info', "ssh tunnel still running - terminating now with SIGKILL\n");
96 kill 9, $cpid;
97 sleep 1;
61b04c6d
TL
98
99 $self->log('err', "ssh tunnel child process (PID $cpid) couldn't be collected\n")
100 if !&$collect_child_process();
1ef75254
DM
101}
102
e0eb1f76
FG
103sub read_tunnel {
104 my ($self, $tunnel, $timeout) = @_;
105
106 $timeout = 60 if !defined($timeout);
107
108 my $reader = $tunnel->{reader};
109
110 my $output;
111 eval {
112 PVE::Tools::run_with_timeout($timeout, sub { $output = <$reader>; });
113 };
114 die "reading from tunnel failed: $@\n" if $@;
115
116 chomp $output;
117
118 return $output;
119}
120
121sub write_tunnel {
122 my ($self, $tunnel, $timeout, $command) = @_;
123
124 $timeout = 60 if !defined($timeout);
125
126 my $writer = $tunnel->{writer};
127
128 eval {
129 PVE::Tools::run_with_timeout($timeout, sub {
130 print $writer "$command\n";
131 $writer->flush();
132 });
133 };
134 die "writing to tunnel failed: $@\n" if $@;
bcb51ae8
FG
135
136 if ($tunnel->{version} && $tunnel->{version} >= 1) {
137 my $res = eval { $self->read_tunnel($tunnel, 10); };
138 die "no reply to command '$command': $@\n" if $@;
139
140 if ($res eq 'OK') {
141 return;
142 } else {
143 die "tunnel replied '$res' to command '$command'\n";
144 }
145 }
e0eb1f76
FG
146}
147
1e3baf05 148sub fork_tunnel {
1c9d54bf 149 my ($self, $tunnel_addr) = @_;
1e3baf05 150
7827de41
ML
151 my @localtunnelinfo = ();
152 foreach my $addr (@$tunnel_addr) {
153 push @localtunnelinfo, '-L', $addr;
154 }
5bc1e039 155
d7b1b24b 156 my $cmd = [@{$self->{rem_ssh}}, '-o ExitOnForwardFailure=yes', @localtunnelinfo, '/usr/sbin/qm', 'mtunnel' ];
19672434 157
46a84fd4 158 my $tunnel = $self->fork_command_pipe($cmd);
1e3baf05 159
19672434 160 eval {
e0eb1f76 161 my $helo = $self->read_tunnel($tunnel, 60);
1e3baf05 162 die "no reply\n" if !$helo;
1ef75254 163 die "no quorum on target node\n" if $helo =~ m/^no quorum$/;
19672434 164 die "got strange reply from mtunnel ('$helo')\n"
1e3baf05
DM
165 if $helo !~ m/^tunnel online$/;
166 };
167 my $err = $@;
168
58cbe639
FG
169 eval {
170 my $ver = $self->read_tunnel($tunnel, 10);
171 if ($ver =~ /^ver (\d+)$/) {
172 $tunnel->{version} = $1;
173 $self->log('info', "ssh tunnel $ver\n");
174 } else {
175 $err = "received invalid tunnel version string '$ver'\n" if !$err;
176 }
177 };
178
1e3baf05 179 if ($err) {
46a84fd4 180 $self->finish_command_pipe($tunnel);
1e3baf05
DM
181 die "can't open migration tunnel - $err";
182 }
183 return $tunnel;
184}
185
19672434 186sub finish_tunnel {
16e903f2 187 my ($self, $tunnel) = @_;
1e3baf05 188
e0eb1f76 189 eval { $self->write_tunnel($tunnel, 30, 'quit'); };
1e3baf05 190 my $err = $@;
19672434 191
97439670 192 $self->finish_command_pipe($tunnel, 30);
19672434 193
1c9d54bf
TL
194 if ($tunnel->{sock_addr}) {
195 # ssh does not clean up on local host
7827de41 196 my $cmd = ['rm', '-f', @{$tunnel->{sock_addr}}]; #
1c9d54bf
TL
197 PVE::Tools::run_command($cmd);
198
199 # .. and just to be sure check on remote side
200 unshift @{$cmd}, @{$self->{rem_ssh}};
201 PVE::Tools::run_command($cmd);
202 }
203
1e3baf05
DM
204 die $err if $err;
205}
206
16e903f2
DM
207sub lock_vm {
208 my ($self, $vmid, $code, @param) = @_;
f5eb281a 209
ffda963f 210 return PVE::QemuConfig->lock_config($vmid, $code, @param);
16e903f2 211}
ff1a2432 212
16e903f2
DM
213sub prepare {
214 my ($self, $vmid) = @_;
ff1a2432 215
16e903f2 216 my $online = $self->{opts}->{online};
3ea94c60 217
16e903f2 218 $self->{storecfg} = PVE::Storage::config();
3ea94c60 219
e1fc368d 220 # test if VM exists
ffda963f 221 my $conf = $self->{vmconf} = PVE::QemuConfig->load_config($vmid);
3ea94c60 222
ffda963f 223 PVE::QemuConfig->check_lock($conf);
3ea94c60 224
16e903f2
DM
225 my $running = 0;
226 if (my $pid = PVE::QemuServer::check_running($vmid)) {
b6adff33 227 die "can't migrate running VM without --online\n" if !$online;
16e903f2 228 $running = $pid;
42dbd2ee 229
3392d6ca 230 $self->{forcemachine} = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
7bac824e 231
58c64ad5
SR
232 # To support custom CPU types, we keep QEMU's "-cpu" parameter intact.
233 # Since the parameter itself contains no reference to a custom model,
234 # this makes migration independent of changes to "cpu-models.conf".
235 if ($conf->{cpu}) {
236 my $cpuconf = PVE::QemuServer::CPUConfig::parse_cpu_conf_basic($conf->{cpu});
237 if ($cpuconf && PVE::QemuServer::CPUConfig::is_custom_model($cpuconf->{cputype})) {
238 $self->{forcecpu} = PVE::QemuServer::CPUConfig::get_cpu_from_running_vm($pid);
239 }
240 }
3ea94c60 241 }
58c64ad5 242
ca6abacf
TM
243 my $loc_res = PVE::QemuServer::check_local_resources($conf, 1);
244 if (scalar @$loc_res) {
16e903f2 245 if ($self->{running} || !$self->{opts}->{force}) {
ca6abacf 246 die "can't migrate VM which uses local devices: " . join(", ", @$loc_res) . "\n";
16e903f2
DM
247 } else {
248 $self->log('info', "migrating VM which uses local devices");
249 }
3ea94c60
DM
250 }
251
ff1a2432 252 my $vollist = PVE::QemuServer::get_vm_volumes($conf);
16e903f2 253
73f5ee92 254 my $need_activate = [];
29701766
FG
255 foreach my $volid (@$vollist) {
256 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
257
258 # check if storage is available on both nodes
bf8fc5a3 259 my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $sid);
b74cad8a 260
29701766 261 my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid);
b74cad8a 262 PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node});
73f5ee92
FG
263
264 if ($scfg->{shared}) {
265 # PVE::Storage::activate_storage checks this for non-shared storages
266 my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
267 warn "Used shared storage '$sid' is not online on source node!\n"
268 if !$plugin->check_connection($sid, $scfg);
269 } else {
270 # only activate if not shared
9e93a63f 271 next if ($volid =~ m/vm-\d+-cloudinit/);
73f5ee92
FG
272 push @$need_activate, $volid;
273 }
29701766 274 }
3ea94c60 275
73f5ee92
FG
276 # activate volumes
277 PVE::Storage::activate_volumes($self->{storecfg}, $need_activate);
278
3ea94c60 279 # test ssh connection
16e903f2
DM
280 my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
281 eval { $self->cmd_quiet($cmd); };
3ea94c60 282 die "Can't connect to destination address using public key\n" if $@;
ff1a2432 283
16e903f2 284 return $running;
3ea94c60
DM
285}
286
287sub sync_disks {
16e903f2
DM
288 my ($self, $vmid) = @_;
289
16e903f2
DM
290 my $conf = $self->{vmconf};
291
dabf2473 292 # local volumes which have been copied
37666e4c 293 # and their old_id => new_id pairs
16e903f2 294 $self->{volumes} = [];
37666e4c 295 $self->{volume_map} = {};
3ea94c60 296
b10afa31 297 my $storecfg = $self->{storecfg};
3ea94c60
DM
298 eval {
299
dabf2473
FG
300 # found local volumes and their origin
301 my $local_volumes = {};
5bf7f0f1
FG
302 my $local_volumes_errors = {};
303 my $other_errors = [];
304 my $abort = 0;
3ea94c60 305
5bf7f0f1
FG
306 my $log_error = sub {
307 my ($msg, $volid) = @_;
308
309 if (defined($volid)) {
310 $local_volumes_errors->{$volid} = $msg;
311 } else {
312 push @$other_errors, $msg;
313 }
314 $abort = 1;
315 };
316
b10afa31 317 my @sids = PVE::Storage::storage_ids($storecfg);
86638cc2 318 foreach my $storeid (@sids) {
b10afa31 319 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
86638cc2 320 next if $scfg->{shared};
b10afa31 321 next if !PVE::Storage::storage_check_enabled($storecfg, $storeid, undef, 1);
373ea579 322
86638cc2 323 # get list from PVE::Storage (for unused volumes)
b10afa31 324 my $dl = PVE::Storage::vdisk_list($storecfg, $storeid, $vmid);
89719f98
FG
325
326 next if @{$dl->{$storeid}} == 0;
327
bf8fc5a3 328 my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $storeid);
86638cc2 329 # check if storage is available on target node
b10afa31 330 PVE::Storage::storage_check_node($storecfg, $targetsid, $self->{node});
89719f98 331
bf8fc5a3
FG
332 # grandfather in existing mismatches
333 if ($targetsid ne $storeid) {
334 my $target_scfg = PVE::Storage::storage_config($storecfg, $targetsid);
335 die "content type 'images' is not available on storage '$targetsid'\n"
336 if !$target_scfg->{content}->{images};
337 }
338
86638cc2 339 PVE::Storage::foreach_volid($dl, sub {
5eca0c36 340 my ($volid, $sid, $volinfo) = @_;
80b2cbd1 341
6f58fce9 342 $local_volumes->{$volid}->{ref} = 'storage';
5eca0c36
FE
343
344 # If with_snapshots is not set for storage migrate, it tries to use
345 # a raw+size stream, but on-the-fly conversion from qcow2 to raw+size
346 # back to qcow2 is currently not possible.
347 $local_volumes->{$volid}->{snapshots} = ($volinfo->{format} =~ /^(?:qcow2|vmdk)$/);
0ad295f9 348 $local_volumes->{$volid}->{format} = $volinfo->{format};
86638cc2
FG
349 });
350 }
3ea94c60 351
88126be3
FG
352 my $rep_cfg = PVE::ReplicationConfig->new();
353 my $replication_jobcfg = $rep_cfg->find_local_replication_job($vmid, $self->{node});
2cd808d3
TL
354 my $replicatable_volumes = !$replication_jobcfg ? {}
355 : PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 1);
b9f44d27 356
3629c19d 357 my $test_volid = sub {
aee6abe5 358 my ($volid, $attr) = @_;
3ea94c60 359
5bf7f0f1 360 if ($volid =~ m|^/|) {
ec82e3ee 361 return if $attr->{shared};
6f58fce9 362 $local_volumes->{$volid}->{ref} = 'config';
5bf7f0f1
FG
363 die "local file/device\n";
364 }
3ea94c60 365
aee6abe5
DM
366 my $snaprefs = $attr->{referenced_in_snapshot};
367
368 if ($attr->{cdrom}) {
5bf7f0f1
FG
369 if ($volid eq 'cdrom') {
370 my $msg = "can't migrate local cdrom drive";
5009a8c7 371 if (defined($snaprefs) && !$attr->{referenced_in_config}) {
aee6abe5 372 my $snapnames = join(', ', sort keys %$snaprefs);
5009a8c7 373 $msg .= " (referenced in snapshot - $snapnames)";
aee6abe5 374 }
5bf7f0f1
FG
375 &$log_error("$msg\n");
376 return;
377 }
3ea94c60 378 return if $volid eq 'none';
3ea94c60
DM
379 }
380
381 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
382
bf8fc5a3 383 my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $sid);
16e903f2 384 # check if storage is available on both nodes
b10afa31
TL
385 my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
386 PVE::Storage::storage_check_node($storecfg, $targetsid, $self->{node});
3ea94c60
DM
387
388 return if $scfg->{shared};
389
6f58fce9 390 $local_volumes->{$volid}->{ref} = $attr->{referenced_in_config} ? 'config' : 'snapshot';
d62fcf74 391
9e93a63f
ML
392 if ($attr->{cdrom}) {
393 if ($volid =~ /vm-\d+-cloudinit/) {
394 $local_volumes->{$volid}->{ref} = 'generated';
395 return;
396 }
397 die "local cdrom image\n";
398 }
3629c19d 399
b10afa31 400 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
3ea94c60 401
5bf7f0f1 402 die "owned by other VM (owner = VM $owner)\n"
b10afa31 403 if !$owner || ($owner != $vmid);
3ea94c60 404
aee6abe5 405 if (defined($snaprefs)) {
5eca0c36
FE
406 $local_volumes->{$volid}->{snapshots} = 1;
407
3629c19d
DM
408 # we cannot migrate shapshots on local storage
409 # exceptions: 'zfspool' or 'qcow2' files (on directory storage)
410
b74cad8a 411 die "online storage migration not possible if snapshot exists\n" if $self->{running};
0ad295f9 412 if (!($scfg->{type} eq 'zfspool' || $local_volumes->{$volid}->{format} eq 'qcow2')) {
5bf7f0f1 413 die "non-migratable snapshot exists\n";
3629c19d 414 }
3629c19d 415 }
3a7bc9e2
FG
416
417 die "referenced by linked clone(s)\n"
b10afa31 418 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
3629c19d
DM
419 };
420
aee6abe5
DM
421 PVE::QemuServer::foreach_volid($conf, sub {
422 my ($volid, $attr) = @_;
423 eval { $test_volid->($volid, $attr); };
424 if (my $err = $@) {
425 &$log_error($err, $volid);
426 }
427 });
3ea94c60 428
dabf2473 429 foreach my $vol (sort keys %$local_volumes) {
b9f44d27 430 my $type = $replicatable_volumes->{$vol} ? 'local, replicated' : 'local';
6f58fce9
WB
431 my $ref = $local_volumes->{$vol}->{ref};
432 if ($ref eq 'storage') {
b9f44d27 433 $self->log('info', "found $type disk '$vol' (via storage)\n");
6f58fce9 434 } elsif ($ref eq 'config') {
dbc9420b
DM
435 &$log_error("can't live migrate attached local disks without with-local-disks option\n", $vol)
436 if $self->{running} && !$self->{opts}->{"with-local-disks"};
b9f44d27 437 $self->log('info', "found $type disk '$vol' (in current VM config)\n");
6f58fce9 438 } elsif ($ref eq 'snapshot') {
b9f44d27 439 $self->log('info', "found $type disk '$vol' (referenced by snapshot(s))\n");
9e93a63f
ML
440 } elsif ($ref eq 'generated') {
441 $self->log('info', "found generated disk '$vol' (in current VM config)\n");
d62fcf74 442 } else {
b9f44d27 443 $self->log('info', "found $type disk '$vol'\n");
d62fcf74
FG
444 }
445 }
446
5bf7f0f1
FG
447 foreach my $vol (sort keys %$local_volumes_errors) {
448 $self->log('warn', "can't migrate local disk '$vol': $local_volumes_errors->{$vol}");
449 }
450 foreach my $err (@$other_errors) {
451 $self->log('warn', "$err");
452 }
453
5bf7f0f1
FG
454 if ($abort) {
455 die "can't migrate VM - check log\n";
456 }
457
c4d2d6c1 458 # additional checks for local storage
dabf2473 459 foreach my $volid (keys %$local_volumes) {
3ea94c60 460 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
b10afa31 461 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
3ea94c60 462
dad06e20 463 my $migratable = $scfg->{type} =~ /^(?:dir|zfspool|lvmthin|lvm)$/;
c4d2d6c1 464
37a6dc78 465 die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n"
c4d2d6c1 466 if !$migratable;
d5604092 467
c4d2d6c1 468 # image is a linked clone on local storage, se we can't migrate.
b10afa31 469 if (my $basename = (PVE::Storage::parse_volname($storecfg, $volid))[3]) {
c4d2d6c1 470 die "can't migrate '$volid' as it's a clone of '$basename'";
d5604092 471 }
3ea94c60
DM
472 }
473
88126be3 474 if ($replication_jobcfg) {
9b6efe43 475 if ($self->{running}) {
28e6e180
TL
476
477 my $version = PVE::QemuServer::kvm_user_version();
478 if (!min_version($version, 4, 2)) {
479 die "can't live migrate VM with replicated volumes, pve-qemu to old (< 4.2)!\n"
480 }
481
9b6efe43 482 my $live_replicatable_volumes = {};
912792e2 483 PVE::QemuConfig->foreach_volume($conf, sub {
9b6efe43
FG
484 my ($ds, $drive) = @_;
485
486 my $volid = $drive->{file};
487 $live_replicatable_volumes->{$ds} = $volid
488 if defined($replicatable_volumes->{$volid});
489 });
490 foreach my $drive (keys %$live_replicatable_volumes) {
491 my $volid = $live_replicatable_volumes->{$drive};
492
493 my $bitmap = "repl_$drive";
494
495 # start tracking before replication to get full delta + a few duplicates
496 $self->log('info', "$drive: start tracking writes using block-dirty-bitmap '$bitmap'");
497 mon_cmd($vmid, 'block-dirty-bitmap-add', node => "drive-$drive", name => $bitmap);
498
499 # other info comes from target node in phase 2
500 $self->{target_drive}->{$drive}->{bitmap} = $bitmap;
501 }
502 }
e85d01f2 503 $self->log('info', "replicating disk images");
9b6efe43 504
dbc9420b 505 my $start_time = time();
e85d01f2
TL
506 my $logfunc = sub { $self->log('info', shift) };
507 $self->{replicated_volumes} = PVE::Replication::run_replication(
88126be3 508 'PVE::QemuConfig', $replication_jobcfg, $start_time, $start_time, $logfunc);
dbc9420b
DM
509 }
510
68b108ee
SR
511 # sizes in config have to be accurate for remote node to correctly
512 # allocate disks, rescan to be sure
b10afa31 513 my $volid_hash = PVE::QemuServer::scan_volids($storecfg, $vmid);
912792e2 514 PVE::QemuConfig->foreach_volume($conf, sub {
68b108ee 515 my ($key, $drive) = @_;
e0fd2b2f 516 my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
68b108ee
SR
517 if (defined($updated)) {
518 $conf->{$key} = PVE::QemuServer::print_drive($updated);
519 $self->log('info', "size of disk '$updated->{file}' ($key) updated from $old_size to $new_size\n");
520 }
521 });
522
818ce80e
DC
523 # we want to set the efidisk size in the config to the size of the
524 # real OVMF_VARS.fd image, else we can create a too big image, which does not work
525 if (defined($conf->{efidisk0})) {
526 PVE::QemuServer::update_efidisk_size($conf);
527 }
528
e85d01f2
TL
529 $self->log('info', "copying local disk images") if scalar(%$local_volumes);
530
dabf2473 531 foreach my $volid (keys %$local_volumes) {
3ea94c60 532 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
bf8fc5a3 533 my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $sid);
9e93a63f
ML
534 my $ref = $local_volumes->{$volid}->{ref};
535 if ($self->{running} && $ref eq 'config') {
b74cad8a 536 push @{$self->{online_local_volumes}}, $volid;
9e93a63f 537 } elsif ($ref eq 'generated') {
9860fe4e 538 die "can't live migrate VM with local cloudinit disk. use a shared storage instead\n" if $self->{running};
9e93a63f
ML
539 # skip all generated volumes but queue them for deletion in phase3_cleanup
540 push @{$self->{volumes}}, $volid;
541 next;
b74cad8a 542 } else {
e85d01f2 543 next if $self->{replicated_volumes}->{$volid};
b74cad8a 544 push @{$self->{volumes}}, $volid;
15a37695 545 my $opts = $self->{opts};
15a37695 546 # use 'migrate' limit for transfer to other node
29eb909e 547 my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', [$targetsid, $sid], $opts->{bwlimit});
15a37695
SI
548 # JSONSchema and get_bandwidth_limit use kbps - storage_migrate bps
549 $bwlimit = $bwlimit * 1024 if defined($bwlimit);
550
1f726e0a
FE
551 my $storage_migrate_opts = {
552 'bwlimit' => $bwlimit,
553 'insecure' => $opts->{migration_type} eq 'insecure',
554 'with_snapshots' => $local_volumes->{$volid}->{snapshots},
555 };
556
37666e4c
FE
557 my $new_volid = PVE::Storage::storage_migrate($storecfg, $volid, $self->{ssh_info},
558 $targetsid, $storage_migrate_opts);
559
560 $self->{volume_map}->{$volid} = $new_volid;
561 $self->log('info', "volume '$volid' is '$new_volid' on the target\n");
b74cad8a 562 }
3ea94c60
DM
563 }
564 };
565 die "Failed to sync data - $@" if $@;
566}
567
b74cad8a
AD
568sub cleanup_remotedisks {
569 my ($self) = @_;
570
571 foreach my $target_drive (keys %{$self->{target_drive}}) {
6a039d06
FG
572 my $drivestr = $self->{target_drive}->{$target_drive}->{drivestr};
573 next if !defined($drivestr);
574
575 my $drive = PVE::QemuServer::parse_drive($target_drive, $drivestr);
9b3f5a5c 576
9b6efe43 577 # don't clean up replicated disks!
9b3f5a5c 578 next if defined($self->{replicated_volumes}->{$drive->{file}});
b74cad8a 579
b74cad8a
AD
580 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
581
582 my $cmd = [@{$self->{rem_ssh}}, 'pvesm', 'free', "$storeid:$volname"];
583
584 eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
585 if (my $err = $@) {
586 $self->log('err', $err);
587 $self->{errors} = 1;
588 }
589 }
590}
591
9b6efe43
FG
592sub cleanup_bitmaps {
593 my ($self) = @_;
7f5fb49a 594 foreach my $drive (keys %{$self->{target_drive}}) {
9b6efe43
FG
595 my $bitmap = $self->{target_drive}->{$drive}->{bitmap};
596 next if !$bitmap;
597 $self->log('info', "$drive: removing block-dirty-bitmap '$bitmap'");
598 mon_cmd($self->{vmid}, 'block-dirty-bitmap-remove', node => "drive-$drive", name => $bitmap);
599 }
600}
601
1e3baf05 602sub phase1 {
16e903f2 603 my ($self, $vmid) = @_;
1e3baf05 604
16e903f2 605 $self->log('info', "starting migration of VM $vmid to node '$self->{node}' ($self->{nodeip})");
1e3baf05 606
16e903f2 607 my $conf = $self->{vmconf};
1e3baf05
DM
608
609 # set migrate lock in config file
1858638f 610 $conf->{lock} = 'migrate';
ffda963f 611 PVE::QemuConfig->write_config($vmid, $conf);
1e3baf05 612
16e903f2 613 sync_disks($self, $vmid);
1ef75254 614
68b108ee
SR
615 # sync_disks fixes disk sizes to match their actual size, write changes so
616 # target allocates correct volumes
617 PVE::QemuConfig->write_config($vmid, $conf);
1e3baf05
DM
618};
619
16e903f2
DM
620sub phase1_cleanup {
621 my ($self, $vmid, $err) = @_;
622
623 $self->log('info', "aborting phase 1 - cleanup resources");
624
1858638f
DM
625 my $conf = $self->{vmconf};
626 delete $conf->{lock};
ffda963f 627 eval { PVE::QemuConfig->write_config($vmid, $conf) };
16e903f2
DM
628 if (my $err = $@) {
629 $self->log('err', $err);
630 }
f5eb281a 631
16e903f2
DM
632 if ($self->{volumes}) {
633 foreach my $volid (@{$self->{volumes}}) {
634 $self->log('err', "found stale volume copy '$volid' on node '$self->{node}'");
635 # fixme: try to remove ?
636 }
637 }
9b6efe43
FG
638
639 eval { $self->cleanup_bitmaps() };
640 if (my $err =$@) {
641 $self->log('err', $err);
642 }
643
16e903f2
DM
644}
645
1e3baf05 646sub phase2 {
16e903f2 647 my ($self, $vmid) = @_;
1e3baf05 648
16e903f2
DM
649 my $conf = $self->{vmconf};
650
46a84fd4 651 $self->log('info', "starting VM $vmid on remote node '$self->{node}'");
1e3baf05 652
5bc1e039 653 my $raddr;
1e3baf05 654 my $rport;
1c9d54bf 655 my $ruri; # the whole migration dst. URI (protocol:address[:port])
7e8dcf2c
AD
656 my $nodename = PVE::INotify::nodename();
657
19672434 658 ## start on remote node
95a4b4a9
AD
659 my $cmd = [@{$self->{rem_ssh}}];
660
7c14dcae 661 my $spice_ticket;
86b8228b 662 if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
0a13e08e 663 my $res = mon_cmd($vmid, 'query-spice');
7c14dcae 664 $spice_ticket = $res->{ticket};
95a4b4a9
AD
665 }
666
1c9d54bf
TL
667 push @$cmd , 'qm', 'start', $vmid, '--skiplock', '--migratedfrom', $nodename;
668
f1c2a53a 669 my $migration_type = $self->{opts}->{migration_type};
2de2d6f7
TL
670
671 push @$cmd, '--migration_type', $migration_type;
672
673 push @$cmd, '--migration_network', $self->{opts}->{migration_network}
674 if $self->{opts}->{migration_network};
675
676 if ($migration_type eq 'insecure') {
1c9d54bf
TL
677 push @$cmd, '--stateuri', 'tcp';
678 } else {
679 push @$cmd, '--stateuri', 'unix';
680 }
95a4b4a9 681
42668529
DM
682 if ($self->{forcemachine}) {
683 push @$cmd, '--machine', $self->{forcemachine};
684 }
685
58c64ad5
SR
686 if ($self->{forcecpu}) {
687 push @$cmd, '--force-cpu', $self->{forcecpu};
688 }
689
4530494b
TL
690 if ($self->{online_local_volumes}) {
691 push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1');
b74cad8a
AD
692 }
693
86b8228b 694 my $spice_port;
7827de41
ML
695 my $tunnel_addr = [];
696 my $sock_addr = [];
697 # version > 0 for unix socket support
698 my $nbd_protocol_version = 1;
ff09c795
ML
699 # TODO change to 'spice_ticket: <ticket>\n' in 7.0
700 my $input = $spice_ticket ? "$spice_ticket\n" : "\n";
701 $input .= "nbd_protocol_version: $nbd_protocol_version\n";
88126be3
FG
702 foreach my $volid (keys %{$self->{replicated_volumes}}) {
703 $input .= "replicated_volume: $volid\n";
704 }
705
706 my $target_replicated_volumes = {};
86b8228b 707
7c14dcae
DM
708 # Note: We try to keep $spice_ticket secret (do not pass via command line parameter)
709 # instead we pipe it through STDIN
7827de41 710 my $exitcode = PVE::Tools::run_command($cmd, input => $input, outfunc => sub {
1e3baf05
DM
711 my $line = shift;
712
407e0b8b 713 if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
5bc1e039
SP
714 $raddr = $1;
715 $rport = int($2);
1c9d54bf
TL
716 $ruri = "tcp:$raddr:$rport";
717 }
718 elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) {
719 $raddr = $1;
720 die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2;
721 $ruri = "unix:$raddr";
5bc1e039
SP
722 }
723 elsif ($line =~ m/^migration listens on port (\d+)$/) {
724 $raddr = "localhost";
86b8228b 725 $rport = int($1);
1c9d54bf 726 $ruri = "tcp:$raddr:$rport";
5bc1e039 727 }
f3a483b6 728 elsif ($line =~ m/^spice listens on port (\d+)$/) {
86b8228b 729 $spice_port = int($1);
1e3baf05 730 }
769f187d 731 elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/) {
8b02e568 732 my $drivestr = $4;
b74cad8a
AD
733 my $nbd_uri = "nbd:$1:$2:exportname=$3";
734 my $targetdrive = $3;
735 $targetdrive =~ s/drive-//g;
736
8b02e568 737 $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr;
b74cad8a 738 $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri;
7827de41
ML
739 } elsif ($line =~ m!^storage migration listens on nbd:unix:(/run/qemu-server/(\d+)_nbd\.migrate):exportname=(\S+) volume:(\S+)$!) {
740 my $drivestr = $4;
741 die "Destination UNIX socket's VMID does not match source VMID" if $vmid ne $2;
742 my $nbd_unix_addr = $1;
743 my $nbd_uri = "nbd:unix:$nbd_unix_addr:exportname=$3";
744 my $targetdrive = $3;
745 $targetdrive =~ s/drive-//g;
b74cad8a 746
7827de41
ML
747 $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr;
748 $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri;
749 push @$tunnel_addr, "$nbd_unix_addr:$nbd_unix_addr";
750 push @$sock_addr, $nbd_unix_addr;
88126be3
FG
751 } elsif ($line =~ m/^re-using replicated volume: (\S+) - (.*)$/) {
752 my $drive = $1;
753 my $volid = $2;
754 $target_replicated_volumes->{$volid} = $drive;
8bf30c2a
SR
755 } elsif ($line =~ m/^QEMU: (.*)$/) {
756 $self->log('info', "[$self->{node}] $1\n");
b74cad8a 757 }
ab399b7c
AD
758 }, errfunc => sub {
759 my $line = shift;
8bf30c2a 760 $self->log('info', "[$self->{node}] $line");
6e0216d8
SR
761 }, noerr => 1);
762
763 die "remote command failed with exit code $exitcode\n" if $exitcode;
1e3baf05 764
5bc1e039 765 die "unable to detect remote migration address\n" if !$raddr;
1ef75254 766
88126be3
FG
767 if (scalar(keys %$target_replicated_volumes) != scalar(keys %{$self->{replicated_volumes}})) {
768 die "number of replicated disks on source and target node do not match - target node too old?\n"
769 }
770
d296ed08
AD
771 $self->log('info', "start remote tunnel");
772
2de2d6f7 773 if ($migration_type eq 'secure') {
1c9d54bf
TL
774
775 if ($ruri =~ /^unix:/) {
54323eed 776 unlink $raddr;
7827de41
ML
777 push @$tunnel_addr, "$raddr:$raddr";
778 $self->{tunnel} = $self->fork_tunnel($tunnel_addr);
779 push @$sock_addr, $raddr;
1c9d54bf
TL
780
781 my $unix_socket_try = 0; # wait for the socket to become ready
7827de41 782 while ($unix_socket_try <= 100) {
1c9d54bf 783 $unix_socket_try++;
7827de41
ML
784 my $available = 0;
785 foreach my $sock (@$sock_addr) {
786 if (-S $sock) {
787 $available++;
788 }
789 }
790
791 if ($available == @$sock_addr) {
792 last;
1c9d54bf
TL
793 }
794
795 usleep(50000);
796 }
7827de41
ML
797 if ($unix_socket_try > 100) {
798 $self->{errors} = 1;
799 $self->finish_tunnel($self->{tunnel});
800 die "Timeout, migration socket $ruri did not get ready";
801 }
1c9d54bf
TL
802
803 } elsif ($ruri =~ /^tcp:/) {
e858e9d2
TL
804 my $tunnel_addr;
805 if ($raddr eq "localhost") {
806 # for backwards compatibility with older qemu-server versions
807 my $pfamily = PVE::Tools::get_host_address_family($nodename);
808 my $lport = PVE::Tools::next_migrate_port($pfamily);
809 $tunnel_addr = "$lport:localhost:$rport";
810 }
1c9d54bf 811
e858e9d2 812 $self->{tunnel} = $self->fork_tunnel($tunnel_addr);
1c9d54bf
TL
813
814 } else {
815 die "unsupported protocol in migration URI: $ruri\n";
816 }
d296ed08
AD
817 } else {
818 #fork tunnel for insecure migration, to send faster commands like resume
819 $self->{tunnel} = $self->fork_tunnel();
1c9d54bf 820 }
7827de41 821 $self->{tunnel}->{sock_addr} = $sock_addr if (@$sock_addr);
1e3baf05 822
1e3baf05 823 my $start = time();
b74cad8a 824
d189e590
SI
825 my $opt_bwlimit = $self->{opts}->{bwlimit};
826
c7789f54 827 if (defined($self->{online_local_volumes})) {
b74cad8a
AD
828 $self->{storage_migration} = 1;
829 $self->{storage_migration_jobs} = {};
830 $self->log('info', "starting storage migration");
831
bd2d5fe6 832 die "The number of local disks does not match between the source and the destination.\n"
3b4cf0f0 833 if (scalar(keys %{$self->{target_drive}}) != scalar @{$self->{online_local_volumes}});
b74cad8a 834 foreach my $drive (keys %{$self->{target_drive}}){
d189e590
SI
835 my $target = $self->{target_drive}->{$drive};
836 my $nbd_uri = $target->{nbd_uri};
683ab654 837
1764fa05 838 my $source_drive = PVE::QemuServer::parse_drive($drive, $conf->{$drive});
683ab654
FG
839 my $target_drive = PVE::QemuServer::parse_drive($drive, $target->{drivestr});
840
97ece9dd
FE
841 my $source_volid = $source_drive->{file};
842 my $target_volid = $target_drive->{file};
843
844 my $source_sid = PVE::Storage::Plugin::parse_volume_id($source_volid);
845 my $target_sid = PVE::Storage::Plugin::parse_volume_id($target_volid);
683ab654 846
29eb909e 847 my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', [$source_sid, $target_sid], $opt_bwlimit);
9b6efe43 848 my $bitmap = $target->{bitmap};
d189e590 849
d108cb1e 850 $self->log('info', "$drive: start migration to $nbd_uri");
9b6efe43 851 PVE::QemuServer::qemu_drive_mirror($vmid, $drive, $nbd_uri, $vmid, undef, $self->{storage_migration_jobs}, 'skip', undef, $bwlimit, $bitmap);
97ece9dd
FE
852
853 $self->{volume_map}->{$source_volid} = $target_volid;
854 $self->log('info', "volume '$source_volid' is '$target_volid' on the target\n");
b74cad8a
AD
855 }
856 }
857
1c9d54bf 858 $self->log('info', "starting online/live migration on $ruri");
5bc1e039 859 $self->{livemigration} = 1;
e18b0b99 860
3beb415b
AD
861 # load_defaults
862 my $defaults = PVE::QemuServer::load_defaults();
863
485449e3
SR
864 $self->log('info', "set migration_caps");
865 eval {
866 PVE::QemuServer::set_migration_caps($vmid);
867 };
868 warn $@ if $@;
869
870 my $qemu_migrate_params = {};
871
ddd664d7
SI
872 # migrate speed can be set via bwlimit (datacenter.cfg and API) and via the
873 # migrate_speed parameter in qm.conf - take the lower of the two.
29eb909e 874 my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', undef, $opt_bwlimit) // 0;
ddd664d7
SI
875 my $migrate_speed = $conf->{migrate_speed} // $bwlimit;
876 # migrate_speed is in MB/s, bwlimit in KB/s
877 $migrate_speed *= 1024;
878
879 $migrate_speed = ($bwlimit < $migrate_speed) ? $bwlimit : $migrate_speed;
880
881 # always set migrate speed (overwrite kvm default of 32m) we set a very high
882 # default of 8192m which is basically unlimited
883 $migrate_speed ||= ($defaults->{migrate_speed} || 8192) * 1024;
884
885 # qmp takes migrate_speed in B/s.
886 $migrate_speed *= 1024;
485449e3
SR
887 $self->log('info', "migration speed limit: $migrate_speed B/s");
888 $qemu_migrate_params->{'max-bandwidth'} = int($migrate_speed);
3beb415b
AD
889
890 my $migrate_downtime = $defaults->{migrate_downtime};
891 $migrate_downtime = $conf->{migrate_downtime} if defined($conf->{migrate_downtime});
c05f1b33
SR
892 # migrate-set-parameters expects limit in ms
893 $migrate_downtime *= 1000;
894 $self->log('info', "migration downtime limit: $migrate_downtime ms");
895 $qemu_migrate_params->{'downtime-limit'} = int($migrate_downtime);
3beb415b 896
171ed95c
EK
897 # set cachesize to 10% of the total memory
898 my $memory = $conf->{memory} || $defaults->{memory};
899 my $cachesize = int($memory * 1048576 / 10);
50d8dd5d
AD
900 $cachesize = round_powerof2($cachesize);
901
485449e3
SR
902 $self->log('info', "migration cachesize: $cachesize B");
903 $qemu_migrate_params->{'xbzrle-cache-size'} = int($cachesize);
904
905 $self->log('info', "set migration parameters");
e18b0b99 906 eval {
485449e3 907 mon_cmd($vmid, "migrate-set-parameters", %{$qemu_migrate_params});
e18b0b99 908 };
485449e3 909 $self->log('info', "migrate-set-parameters error: $@") if $@;
f34d1466 910
86b8228b 911 if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
95a4b4a9
AD
912 my $rpcenv = PVE::RPCEnvironment::get();
913 my $authuser = $rpcenv->get_user();
914
86b8228b 915 my (undef, $proxyticket) = PVE::AccessControl::assemble_spice_ticket($authuser, $vmid, $self->{node});
95a4b4a9 916
86b8228b 917 my $filename = "/etc/pve/nodes/$self->{node}/pve-ssl.pem";
769f187d 918 my $subject = PVE::AccessControl::read_x509_subject_spice($filename);
95a4b4a9
AD
919
920 $self->log('info', "spice client_migrate_info");
921
922 eval {
0a13e08e 923 mon_cmd($vmid, "client_migrate_info", protocol => 'spice',
ccab68c2 924 hostname => $proxyticket, 'port' => 0, 'tls-port' => $spice_port,
86b8228b 925 'cert-subject' => $subject);
95a4b4a9
AD
926 };
927 $self->log('info', "client_migrate_info error: $@") if $@;
928
929 }
930
f34d1466 931 $self->log('info', "start migrate command to $ruri");
5a7835f5 932 eval {
0a13e08e 933 mon_cmd($vmid, "migrate", uri => $ruri);
5a7835f5
AD
934 };
935 my $merr = $@;
1c9d54bf 936 $self->log('info', "migrate uri => $ruri failed: $merr") if $merr;
1e3baf05 937
a05b47a8 938 my $lstat = 0;
4305207d 939 my $usleep = 1000000;
e52bd94c 940 my $i = 0;
b0b756c1 941 my $err_count = 0;
865ef132
SP
942 my $lastrem = undef;
943 my $downtimecounter = 0;
1e3baf05 944 while (1) {
e52bd94c
AD
945 $i++;
946 my $avglstat = $lstat/$i if $lstat;
947
b0b756c1
DM
948 usleep($usleep);
949 my $stat;
950 eval {
0a13e08e 951 $stat = mon_cmd($vmid, "query-migrate");
b0b756c1
DM
952 };
953 if (my $err = $@) {
954 $err_count++;
955 warn "query migrate failed: $err\n";
f34d1466 956 $self->log('info', "query migrate failed: $err");
b0b756c1
DM
957 if ($err_count <= 5) {
958 usleep(1000000);
959 next;
960 }
961 die "too many query migrate failures - aborting\n";
962 }
985a5f48 963
f34d1466 964 if (defined($stat->{status}) && $stat->{status} =~ m/^(setup)$/im) {
985a5f48
AD
965 sleep(1);
966 next;
967 }
968
f34d1466 969 if (defined($stat->{status}) && $stat->{status} =~ m/^(active|completed|failed|cancelled)$/im) {
d68afb26 970 $merr = undef;
b0b756c1 971 $err_count = 0;
5a7835f5 972 if ($stat->{status} eq 'completed') {
1e3baf05
DM
973 my $delay = time() - $start;
974 if ($delay > 0) {
171ed95c 975 my $mbps = sprintf "%.2f", $memory / $delay;
135007c0
AD
976 my $downtime = $stat->{downtime} || 0;
977 $self->log('info', "migration speed: $mbps MB/s - downtime $downtime ms");
1e3baf05
DM
978 }
979 }
f5eb281a 980
5a7835f5 981 if ($stat->{status} eq 'failed' || $stat->{status} eq 'cancelled') {
f34d1466 982 $self->log('info', "migration status error: $stat->{status}");
1e3baf05
DM
983 die "aborting\n"
984 }
985
a05b47a8
DM
986 if ($stat->{status} ne 'active') {
987 $self->log('info', "migration status: $stat->{status}");
988 last;
989 }
990
991 if ($stat->{ram}->{transferred} ne $lstat) {
992 my $trans = $stat->{ram}->{transferred} || 0;
993 my $rem = $stat->{ram}->{remaining} || 0;
994 my $total = $stat->{ram}->{total} || 0;
e18b0b99
AD
995 my $xbzrlecachesize = $stat->{"xbzrle-cache"}->{"cache-size"} || 0;
996 my $xbzrlebytes = $stat->{"xbzrle-cache"}->{"bytes"} || 0;
997 my $xbzrlepages = $stat->{"xbzrle-cache"}->{"pages"} || 0;
998 my $xbzrlecachemiss = $stat->{"xbzrle-cache"}->{"cache-miss"} || 0;
999 my $xbzrleoverflow = $stat->{"xbzrle-cache"}->{"overflow"} || 0;
4305207d
FG
1000 # reduce sleep if remainig memory is lower than the average transfer speed
1001 $usleep = 100000 if $avglstat && $rem < $avglstat;
a05b47a8
DM
1002
1003 $self->log('info', "migration status: $stat->{status} (transferred ${trans}, " .
0302101c 1004 "remaining ${rem}), total ${total})");
e18b0b99 1005
2e787b18
SP
1006 if (${xbzrlecachesize}) {
1007 $self->log('info', "migration xbzrle cachesize: ${xbzrlecachesize} transferred ${xbzrlebytes} pages ${xbzrlepages} cachemiss ${xbzrlecachemiss} overflow ${xbzrleoverflow}");
1008 }
1009
865ef132
SP
1010 if (($lastrem && $rem > $lastrem ) || ($rem == 0)) {
1011 $downtimecounter++;
1012 }
1013 $lastrem = $rem;
1014
1015 if ($downtimecounter > 5) {
1016 $downtimecounter = 0;
1017 $migrate_downtime *= 2;
c05f1b33 1018 $self->log('info', "auto-increased downtime to continue migration: $migrate_downtime ms");
865ef132 1019 eval {
c05f1b33
SR
1020 # migrate-set-parameters does not touch values not
1021 # specified, so this only changes downtime-limit
1022 mon_cmd($vmid, "migrate-set-parameters", 'downtime-limit' => int($migrate_downtime));
865ef132 1023 };
c05f1b33 1024 $self->log('info', "migrate-set-parameters error: $@") if $@;
865ef132
SP
1025 }
1026
a05b47a8
DM
1027 }
1028
865ef132 1029
a05b47a8 1030 $lstat = $stat->{ram}->{transferred};
370b05e7 1031
1e3baf05 1032 } else {
d68afb26 1033 die $merr if $merr;
5a7835f5 1034 die "unable to parse migration status '$stat->{status}' - aborting\n";
1e3baf05 1035 }
a05b47a8 1036 }
1e3baf05 1037}
16e903f2 1038
c04b5b04
AD
1039sub phase2_cleanup {
1040 my ($self, $vmid, $err) = @_;
1041
af30308f
DM
1042 return if !$self->{errors};
1043 $self->{phase2errors} = 1;
1044
c04b5b04
AD
1045 $self->log('info', "aborting phase 2 - cleanup resources");
1046
19168b91
SP
1047 $self->log('info', "migrate_cancel");
1048 eval {
0a13e08e 1049 mon_cmd($vmid, "migrate_cancel");
19168b91
SP
1050 };
1051 $self->log('info', "migrate_cancel error: $@") if $@;
1052
c04b5b04
AD
1053 my $conf = $self->{vmconf};
1054 delete $conf->{lock};
ffda963f 1055 eval { PVE::QemuConfig->write_config($vmid, $conf) };
c04b5b04
AD
1056 if (my $err = $@) {
1057 $self->log('err', $err);
1058 }
1059
af30308f 1060 # cleanup ressources on target host
3b4cf0f0 1061 if ($self->{storage_migration}) {
b74cad8a
AD
1062 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $self->{storage_migration_jobs}) };
1063 if (my $err = $@) {
1064 $self->log('err', $err);
1065 }
9b3f5a5c 1066 }
b74cad8a 1067
9b3f5a5c
FG
1068 eval { $self->cleanup_bitmaps() };
1069 if (my $err =$@) {
1070 $self->log('err', $err);
b74cad8a
AD
1071 }
1072
af30308f 1073 my $nodename = PVE::INotify::nodename();
370b05e7 1074
af30308f
DM
1075 my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
1076 eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
1077 if (my $err = $@) {
1078 $self->log('err', $err);
1079 $self->{errors} = 1;
1080 }
386c6ba7 1081
9b3f5a5c
FG
1082 # cleanup after stopping, otherwise disks might be in-use by target VM!
1083 eval { PVE::QemuMigrate::cleanup_remotedisks($self) };
1084 if (my $err = $@) {
1085 $self->log('err', $err);
1086 }
1087
1088
386c6ba7
WL
1089 if ($self->{tunnel}) {
1090 eval { finish_tunnel($self, $self->{tunnel}); };
1091 if (my $err = $@) {
1092 $self->log('err', $err);
1093 $self->{errors} = 1;
1094 }
1095 }
c04b5b04
AD
1096}
1097
16e903f2
DM
1098sub phase3 {
1099 my ($self, $vmid) = @_;
f5eb281a 1100
16e903f2 1101 my $volids = $self->{volumes};
af30308f 1102 return if $self->{phase2errors};
16e903f2
DM
1103
1104 # destroy local copies
1105 foreach my $volid (@$volids) {
46883f80
DM
1106 eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
1107 if (my $err = $@) {
1108 $self->log('err', "removing local copy of '$volid' failed - $err");
1109 $self->{errors} = 1;
1110 last if $err =~ /^interrupted by signal$/;
16e903f2
DM
1111 }
1112 }
16e903f2
DM
1113}
1114
1115sub phase3_cleanup {
1116 my ($self, $vmid, $err) = @_;
1117
1118 my $conf = $self->{vmconf};
af30308f 1119 return if $self->{phase2errors};
16e903f2 1120
1d5aaa1d
FG
1121 my $tunnel = $self->{tunnel};
1122
b74cad8a 1123 if ($self->{storage_migration}) {
db1f8b39 1124 # finish block-job with block-job-cancel, to disconnect source VM from NBD
1e0074c4
TL
1125 # to avoid it trying to re-establish it. We are in blockjob ready state,
1126 # thus, this command changes to it to blockjob complete (see qapi docs)
db1f8b39 1127 eval { PVE::QemuServer::qemu_drive_mirror_monitor($vmid, undef, $self->{storage_migration_jobs}, 'cancel'); };
b74cad8a
AD
1128
1129 if (my $err = $@) {
1130 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $self->{storage_migration_jobs}) };
1131 eval { PVE::QemuMigrate::cleanup_remotedisks($self) };
40a572f7 1132 die "Failed to complete storage migration: $err\n";
b74cad8a
AD
1133 }
1134 }
1135
97ece9dd
FE
1136 if ($self->{volume_map}) {
1137 PVE::QemuConfig->update_volume_ids($conf, $self->{volume_map});
37666e4c
FE
1138 PVE::QemuConfig->write_config($vmid, $conf);
1139 }
1140
dbc9420b 1141 # transfer replication state before move config
4bdd20ab 1142 $self->transfer_replication_state() if $self->{replicated_volumes};
dbc9420b 1143
b8d20802 1144 # move config to remote node
ffda963f
FG
1145 my $conffile = PVE::QemuConfig->config_file($vmid);
1146 my $newconffile = PVE::QemuConfig->config_file($vmid, $self->{node});
b8d20802
AD
1147
1148 die "Failed to move config to node '$self->{node}' - rename failed: $!\n"
1149 if !rename($conffile, $newconffile);
1150
4bdd20ab 1151 $self->switch_replication_job_target() if $self->{replicated_volumes};
dbc9420b 1152
5bc1e039 1153 if ($self->{livemigration}) {
504105c6
FG
1154 if ($self->{storage_migration}) {
1155 # stop nbd server on remote vm - requirement for resume since 2.9
1156 my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
1157
1158 eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
1159 if (my $err = $@) {
1160 $self->log('err', $err);
1161 $self->{errors} = 1;
1162 }
1163 }
1d5aaa1d 1164
877e2ea7 1165 # config moved and nbd server stopped - now we can resume vm on target
1d5aaa1d
FG
1166 if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) {
1167 eval {
1168 $self->write_tunnel($tunnel, 30, "resume $vmid");
1169 };
1170 if (my $err = $@) {
1171 $self->log('err', $err);
1172 $self->{errors} = 1;
1173 }
1174 } else {
1175 my $cmd = [@{$self->{rem_ssh}}, 'qm', 'resume', $vmid, '--skiplock', '--nocheck'];
1176 my $logf = sub {
1177 my $line = shift;
1178 $self->log('err', $line);
1179 };
1180 eval { PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => $logf); };
1181 if (my $err = $@) {
1182 $self->log('err', $err);
1183 $self->{errors} = 1;
1184 }
b67900f1 1185 }
ca662131
SI
1186
1187 if ($self->{storage_migration} && PVE::QemuServer::parse_guest_agent($conf)->{fstrim_cloned_disks} && $self->{running}) {
1188 my $cmd = [@{$self->{rem_ssh}}, 'qm', 'guest', 'cmd', $vmid, 'fstrim'];
1189 eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
1190 }
b67900f1
AD
1191 }
1192
2e7fee87
FG
1193 # close tunnel on successful migration, on error phase2_cleanup closed it
1194 if ($tunnel) {
1195 eval { finish_tunnel($self, $tunnel); };
1196 if (my $err = $@) {
1197 $self->log('err', $err);
1198 $self->{errors} = 1;
1199 }
1200 }
1201
fd8469f7 1202 eval {
fd8469f7
AD
1203 my $timer = 0;
1204 if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running}) {
1205 $self->log('info', "Waiting for spice server migration");
1206 while (1) {
0a13e08e 1207 my $res = mon_cmd($vmid, 'query-spice');
fd8469f7
AD
1208 last if int($res->{'migrated'}) == 1;
1209 last if $timer > 50;
1210 $timer ++;
1211 usleep(200000);
769f187d 1212 }
fd8469f7
AD
1213 }
1214 };
95a4b4a9 1215
16e903f2
DM
1216 # always stop local VM
1217 eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); };
1218 if (my $err = $@) {
1219 $self->log('err', "stopping vm failed - $err");
1220 $self->{errors} = 1;
1221 }
1222
1223 # always deactivate volumes - avoid lvm LVs to be active on several nodes
1224 eval {
1225 my $vollist = PVE::QemuServer::get_vm_volumes($conf);
1226 PVE::Storage::deactivate_volumes($self->{storecfg}, $vollist);
1227 };
1228 if (my $err = $@) {
1229 $self->log('err', $err);
1230 $self->{errors} = 1;
1231 }
1232
b74cad8a
AD
1233 if($self->{storage_migration}) {
1234 # destroy local copies
1235 my $volids = $self->{online_local_volumes};
1236
1237 foreach my $volid (@$volids) {
9b6efe43
FG
1238 # keep replicated volumes!
1239 next if $self->{replicated_volumes}->{$volid};
1240
b74cad8a
AD
1241 eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
1242 if (my $err = $@) {
1243 $self->log('err', "removing local copy of '$volid' failed - $err");
1244 $self->{errors} = 1;
1245 last if $err =~ /^interrupted by signal$/;
1246 }
1247 }
1248
b74cad8a
AD
1249 }
1250
16e903f2
DM
1251 # clear migrate lock
1252 my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
1253 $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
1254}
1255
1256sub final_cleanup {
1257 my ($self, $vmid) = @_;
1258
1259 # nothing to do
1260}
1261
50d8dd5d
AD
1262sub round_powerof2 {
1263 return 1 if $_[0] < 2;
1264 return 2 << int(log($_[0]-1)/log(2));
1265}
1266
16e903f2 12671;