]> git.proxmox.com Git - qemu-server.git/blame - PVE/QemuMigrate.pm
vm_deviceunplug: raise expection if something fail
[qemu-server.git] / PVE / QemuMigrate.pm
CommitLineData
3ea94c60 1package PVE::QemuMigrate;
1ef75254 2
1e3baf05 3use strict;
3ea94c60 4use warnings;
16e903f2 5use PVE::AbstractMigrate;
3ea94c60 6use IO::File;
1e3baf05 7use IPC::Open2;
3ea94c60 8use PVE::INotify;
f9a971e0 9use PVE::Tools;
3ea94c60 10use PVE::Cluster;
1e3baf05 11use PVE::Storage;
3ea94c60 12use PVE::QemuServer;
e52bd94c 13use Time::HiRes qw( usleep );
95a4b4a9 14use PVE::RPCEnvironment;
1e3baf05 15
16e903f2 16use base qw(PVE::AbstractMigrate);
1e3baf05 17
1ef75254 18sub fork_command_pipe {
46a84fd4 19 my ($self, $cmd) = @_;
19672434 20
1ef75254
DM
21 my $reader = IO::File->new();
22 my $writer = IO::File->new();
23
24 my $orig_pid = $$;
25
26 my $cpid;
27
28 eval { $cpid = open2($reader, $writer, @$cmd); };
29
30 my $err = $@;
31
32 # catch exec errors
33 if ($orig_pid != $$) {
46a84fd4 34 $self->log('err', "can't fork command pipe\n");
19672434
DM
35 POSIX::_exit(1);
36 kill('KILL', $$);
1ef75254
DM
37 }
38
39 die $err if $err;
40
41 return { writer => $writer, reader => $reader, pid => $cpid };
42}
43
19672434 44sub finish_command_pipe {
97439670 45 my ($self, $cmdpipe, $timeout) = @_;
1ef75254
DM
46
47 my $writer = $cmdpipe->{writer};
48 my $reader = $cmdpipe->{reader};
49
50 $writer->close();
51 $reader->close();
52
53 my $cpid = $cmdpipe->{pid};
54
97439670
DM
55 if ($timeout) {
56 for (my $i = 0; $i < $timeout; $i++) {
57 return if !PVE::ProcFSTools::check_process_running($cpid);
58 sleep(1);
59 }
60 }
61
62 $self->log('info', "ssh tunnel still running - terminating now with SIGTERM\n");
63 kill(15, $cpid);
1ef75254 64
97439670
DM
65 # wait again
66 for (my $i = 0; $i < 10; $i++) {
67 return if !PVE::ProcFSTools::check_process_running($cpid);
68 sleep(1);
69 }
70
71 $self->log('info', "ssh tunnel still running - terminating now with SIGKILL\n");
72 kill 9, $cpid;
73 sleep 1;
1ef75254
DM
74}
75
1e3baf05 76sub fork_tunnel {
16e903f2 77 my ($self, $nodeip, $lport, $rport) = @_;
1e3baf05 78
033a0b36 79 my @localtunnelinfo = $lport ? ('-L' , "$lport:localhost:$rport" ) : ();
5bc1e039 80
033a0b36 81 my $cmd = [@{$self->{rem_ssh}}, @localtunnelinfo, 'qm', 'mtunnel' ];
19672434 82
46a84fd4 83 my $tunnel = $self->fork_command_pipe($cmd);
1e3baf05
DM
84
85 my $reader = $tunnel->{reader};
86
87 my $helo;
19672434 88 eval {
17eed025 89 PVE::Tools::run_with_timeout(60, sub { $helo = <$reader>; });
1e3baf05 90 die "no reply\n" if !$helo;
1ef75254 91 die "no quorum on target node\n" if $helo =~ m/^no quorum$/;
19672434 92 die "got strange reply from mtunnel ('$helo')\n"
1e3baf05
DM
93 if $helo !~ m/^tunnel online$/;
94 };
95 my $err = $@;
96
97 if ($err) {
46a84fd4 98 $self->finish_command_pipe($tunnel);
1e3baf05
DM
99 die "can't open migration tunnel - $err";
100 }
101 return $tunnel;
102}
103
19672434 104sub finish_tunnel {
16e903f2 105 my ($self, $tunnel) = @_;
1e3baf05
DM
106
107 my $writer = $tunnel->{writer};
108
19672434 109 eval {
17eed025 110 PVE::Tools::run_with_timeout(30, sub {
1e3baf05
DM
111 print $writer "quit\n";
112 $writer->flush();
19672434 113 });
1e3baf05
DM
114 };
115 my $err = $@;
19672434 116
97439670 117 $self->finish_command_pipe($tunnel, 30);
19672434 118
1e3baf05
DM
119 die $err if $err;
120}
121
16e903f2
DM
122sub lock_vm {
123 my ($self, $vmid, $code, @param) = @_;
f5eb281a 124
16e903f2
DM
125 return PVE::QemuServer::lock_config($vmid, $code, @param);
126}
ff1a2432 127
16e903f2
DM
128sub prepare {
129 my ($self, $vmid) = @_;
ff1a2432 130
16e903f2 131 my $online = $self->{opts}->{online};
3ea94c60 132
16e903f2 133 $self->{storecfg} = PVE::Storage::config();
3ea94c60 134
16e903f2
DM
135 # test is VM exist
136 my $conf = $self->{vmconf} = PVE::QemuServer::load_config($vmid);
3ea94c60 137
16e903f2 138 PVE::QemuServer::check_lock($conf);
3ea94c60 139
16e903f2
DM
140 my $running = 0;
141 if (my $pid = PVE::QemuServer::check_running($vmid)) {
142 die "cant migrate running VM without --online\n" if !$online;
143 $running = $pid;
42668529 144 $self->{forcemachine} = PVE::QemuServer::get_current_qemu_machine($vmid);
3ea94c60
DM
145 }
146
16e903f2
DM
147 if (my $loc_res = PVE::QemuServer::check_local_resources($conf, 1)) {
148 if ($self->{running} || !$self->{opts}->{force}) {
149 die "can't migrate VM which uses local devices\n";
150 } else {
151 $self->log('info', "migrating VM which uses local devices");
152 }
3ea94c60
DM
153 }
154
ff1a2432
DM
155 # activate volumes
156 my $vollist = PVE::QemuServer::get_vm_volumes($conf);
16e903f2
DM
157 PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
158
159 # fixme: check if storage is available on both nodes
3ea94c60
DM
160
161 # test ssh connection
16e903f2
DM
162 my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
163 eval { $self->cmd_quiet($cmd); };
3ea94c60 164 die "Can't connect to destination address using public key\n" if $@;
ff1a2432 165
16e903f2 166 return $running;
3ea94c60
DM
167}
168
169sub sync_disks {
16e903f2
DM
170 my ($self, $vmid) = @_;
171
172 $self->log('info', "copying disk images");
3ea94c60 173
16e903f2
DM
174 my $conf = $self->{vmconf};
175
176 $self->{volumes} = [];
3ea94c60
DM
177
178 my $res = [];
179
180 eval {
181
182 my $volhash = {};
183 my $cdromhash = {};
184
a06c7f7e
DM
185 my $sharedvm = 1;
186
522c8f97
DM
187 my @sids = PVE::Storage::storage_ids($self->{storecfg});
188 foreach my $storeid (@sids) {
189 my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
190 next if $scfg->{shared};
373ea579
DM
191 next if !PVE::Storage::storage_check_enabled($self->{storecfg}, $storeid, undef, 1);
192
80b2cbd1
AD
193 # get list from PVE::Storage (for unused volumes)
194 my $dl = PVE::Storage::vdisk_list($self->{storecfg}, $storeid, $vmid);
195 PVE::Storage::foreach_volid($dl, sub {
196 my ($volid, $sid, $volname) = @_;
197
373ea579 198 # check if storage is available on target node
80b2cbd1
AD
199 PVE::Storage::storage_check_node($self->{storecfg}, $sid, $self->{node});
200
201 $volhash->{$volid} = 1;
a06c7f7e 202 $sharedvm = 0; # there is a non-shared disk
80b2cbd1
AD
203 });
204 }
3ea94c60 205
d5769dc2 206 # and add used, owned/non-shared disks (just to be sure we have all)
3ea94c60 207
d5769dc2
DM
208 PVE::QemuServer::foreach_volid($conf, sub {
209 my ($volid, $is_cdrom) = @_;
3ea94c60 210
3ea94c60
DM
211 return if !$volid;
212
213 die "cant migrate local file/device '$volid'\n" if $volid =~ m|^/|;
214
d5769dc2 215 if ($is_cdrom) {
3ea94c60
DM
216 die "cant migrate local cdrom drive\n" if $volid eq 'cdrom';
217 return if $volid eq 'none';
218 $cdromhash->{$volid} = 1;
219 }
220
221 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
222
16e903f2
DM
223 # check if storage is available on both nodes
224 my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid);
225 PVE::Storage::storage_check_node($self->{storecfg}, $sid, $self->{node});
3ea94c60
DM
226
227 return if $scfg->{shared};
228
229 die "can't migrate local cdrom '$volid'\n" if $cdromhash->{$volid};
230
231 $sharedvm = 0;
232
16e903f2 233 my ($path, $owner) = PVE::Storage::path($self->{storecfg}, $volid);
3ea94c60
DM
234
235 die "can't migrate volume '$volid' - owned by other VM (owner = VM $owner)\n"
16e903f2 236 if !$owner || ($owner != $self->{vmid});
3ea94c60
DM
237
238 $volhash->{$volid} = 1;
239 });
240
16e903f2 241 if ($self->{running} && !$sharedvm) {
3ea94c60
DM
242 die "can't do online migration - VM uses local disks\n";
243 }
244
245 # do some checks first
246 foreach my $volid (keys %$volhash) {
247 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
16e903f2 248 my $scfg = PVE::Storage::storage_config($self->{storecfg}, $sid);
3ea94c60
DM
249
250 die "can't migrate '$volid' - storagy type '$scfg->{type}' not supported\n"
251 if $scfg->{type} ne 'dir';
d5604092 252
b7b1ac9d
DM
253 # if file, check if a backing file exist
254 if (($scfg->{type} eq 'dir') && (!$sharedvm)) {
d5604092 255 my (undef, undef, undef, $parent) = PVE::Storage::volume_size_info($self->{storecfg}, $volid, 1);
b7b1ac9d 256 die "can't migrate '$volid' as it's a clone of '$parent'" if $parent;
d5604092 257 }
3ea94c60
DM
258 }
259
260 foreach my $volid (keys %$volhash) {
261 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
16e903f2
DM
262 push @{$self->{volumes}}, $volid;
263 PVE::Storage::storage_migrate($self->{storecfg}, $volid, $self->{nodeip}, $sid);
3ea94c60
DM
264 }
265 };
266 die "Failed to sync data - $@" if $@;
267}
268
1e3baf05 269sub phase1 {
16e903f2 270 my ($self, $vmid) = @_;
1e3baf05 271
16e903f2 272 $self->log('info', "starting migration of VM $vmid to node '$self->{node}' ($self->{nodeip})");
1e3baf05 273
16e903f2 274 my $conf = $self->{vmconf};
1e3baf05
DM
275
276 # set migrate lock in config file
1858638f
DM
277 $conf->{lock} = 'migrate';
278 PVE::QemuServer::update_config_nolock($vmid, $conf, 1);
1e3baf05 279
16e903f2 280 sync_disks($self, $vmid);
1ef75254 281
1e3baf05
DM
282};
283
16e903f2
DM
284sub phase1_cleanup {
285 my ($self, $vmid, $err) = @_;
286
287 $self->log('info', "aborting phase 1 - cleanup resources");
288
1858638f
DM
289 my $conf = $self->{vmconf};
290 delete $conf->{lock};
291 eval { PVE::QemuServer::update_config_nolock($vmid, $conf, 1) };
16e903f2
DM
292 if (my $err = $@) {
293 $self->log('err', $err);
294 }
f5eb281a 295
16e903f2
DM
296 if ($self->{volumes}) {
297 foreach my $volid (@{$self->{volumes}}) {
298 $self->log('err', "found stale volume copy '$volid' on node '$self->{node}'");
299 # fixme: try to remove ?
300 }
301 }
302}
303
1e3baf05 304sub phase2 {
16e903f2 305 my ($self, $vmid) = @_;
1e3baf05 306
16e903f2
DM
307 my $conf = $self->{vmconf};
308
46a84fd4 309 $self->log('info', "starting VM $vmid on remote node '$self->{node}'");
1e3baf05 310
5bc1e039 311 my $raddr;
1e3baf05 312 my $rport;
7e8dcf2c
AD
313 my $nodename = PVE::INotify::nodename();
314
19672434 315 ## start on remote node
95a4b4a9
AD
316 my $cmd = [@{$self->{rem_ssh}}];
317
7c14dcae 318 my $spice_ticket;
86b8228b 319 if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
95a4b4a9 320 my $res = PVE::QemuServer::vm_mon_cmd($vmid, 'query-spice');
7c14dcae 321 $spice_ticket = $res->{ticket};
95a4b4a9
AD
322 }
323
324 push @$cmd , 'qm', 'start', $vmid, '--stateuri', 'tcp', '--skiplock', '--migratedfrom', $nodename;
325
42668529
DM
326 if ($self->{forcemachine}) {
327 push @$cmd, '--machine', $self->{forcemachine};
328 }
329
86b8228b
DM
330 my $spice_port;
331
7c14dcae
DM
332 # Note: We try to keep $spice_ticket secret (do not pass via command line parameter)
333 # instead we pipe it through STDIN
334 PVE::Tools::run_command($cmd, input => $spice_ticket, outfunc => sub {
1e3baf05
DM
335 my $line = shift;
336
5bc1e039
SP
337 if ($line =~ m/^migration listens on tcp:([\d\.]+|localhost):(\d+)$/) {
338 $raddr = $1;
339 $rport = int($2);
340 }
341 elsif ($line =~ m/^migration listens on port (\d+)$/) {
342 $raddr = "localhost";
86b8228b 343 $rport = int($1);
5bc1e039
SP
344 }
345 elsif ($line =~ m/^spice listens on port (\d+)$/) {
86b8228b 346 $spice_port = int($1);
1e3baf05 347 }
ab399b7c
AD
348 }, errfunc => sub {
349 my $line = shift;
350 $self->log('info', $line);
351 });
1e3baf05 352
5bc1e039 353 die "unable to detect remote migration address\n" if !$raddr;
1ef75254 354
1e3baf05 355 ## create tunnel to remote port
5bc1e039
SP
356 $self->log('info', "starting ssh migration tunnel");
357 my $lport = ($raddr eq "localhost") ? PVE::Tools::next_migrate_port() : undef;
16e903f2 358 $self->{tunnel} = $self->fork_tunnel($self->{nodeip}, $lport, $rport);
1e3baf05 359
1e3baf05 360 my $start = time();
5bc1e039
SP
361 $self->log('info', "starting online/live migration on $raddr:$rport");
362 $self->{livemigration} = 1;
e18b0b99 363
3beb415b
AD
364 # load_defaults
365 my $defaults = PVE::QemuServer::load_defaults();
366
367 # always set migrate speed (overwrite kvm default of 32m)
368 # we set a very hight default of 8192m which is basically unlimited
369 my $migrate_speed = $defaults->{migrate_speed} || 8192;
370 $migrate_speed = $conf->{migrate_speed} || $migrate_speed;
371 $migrate_speed = $migrate_speed * 1048576;
372 $self->log('info', "migrate_set_speed: $migrate_speed");
373 eval {
374 PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate_set_speed", value => int($migrate_speed));
375 };
376 $self->log('info', "migrate_set_speed error: $@") if $@;
377
378 my $migrate_downtime = $defaults->{migrate_downtime};
379 $migrate_downtime = $conf->{migrate_downtime} if defined($conf->{migrate_downtime});
380 if (defined($migrate_downtime)) {
381 $self->log('info', "migrate_set_downtime: $migrate_downtime");
382 eval {
865ef132 383 PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate_set_downtime", value => int($migrate_downtime*100)/100);
3beb415b
AD
384 };
385 $self->log('info', "migrate_set_downtime error: $@") if $@;
386 }
387
e18b0b99 388 eval {
a89fded1 389 PVE::QemuServer::set_migration_caps($vmid);
e18b0b99 390 };
a89fded1 391 warn $@ if $@;
e18b0b99
AD
392
393 #set cachesize 10% of the total memory
394 my $cachesize = int($conf->{memory}*1048576/10);
395 eval {
396 PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate-set-cache-size", value => $cachesize);
397 };
86b8228b
DM
398
399 if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
95a4b4a9
AD
400 my $rpcenv = PVE::RPCEnvironment::get();
401 my $authuser = $rpcenv->get_user();
402
86b8228b 403 my (undef, $proxyticket) = PVE::AccessControl::assemble_spice_ticket($authuser, $vmid, $self->{node});
95a4b4a9 404
86b8228b 405 my $filename = "/etc/pve/nodes/$self->{node}/pve-ssl.pem";
dd25eecf 406 my $subject = PVE::AccessControl::read_x509_subject_spice($filename);
95a4b4a9
AD
407
408 $self->log('info', "spice client_migrate_info");
409
410 eval {
86b8228b
DM
411 PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "client_migrate_info", protocol => 'spice',
412 hostname => $proxyticket, 'tls-port' => $spice_port,
413 'cert-subject' => $subject);
95a4b4a9
AD
414 };
415 $self->log('info', "client_migrate_info error: $@") if $@;
416
417 }
418
5a7835f5 419 eval {
5bc1e039 420 PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate", uri => "tcp:$raddr:$rport");
5a7835f5
AD
421 };
422 my $merr = $@;
5bc1e039 423 $self->log('info', "migrate uri => tcp:$raddr:$rport failed: $merr") if $merr;
1e3baf05 424
a05b47a8 425 my $lstat = 0;
e52bd94c
AD
426 my $usleep = 2000000;
427 my $i = 0;
b0b756c1 428 my $err_count = 0;
865ef132
SP
429 my $lastrem = undef;
430 my $downtimecounter = 0;
1e3baf05 431 while (1) {
e52bd94c
AD
432 $i++;
433 my $avglstat = $lstat/$i if $lstat;
434
b0b756c1
DM
435 usleep($usleep);
436 my $stat;
437 eval {
438 $stat = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "query-migrate");
439 };
440 if (my $err = $@) {
441 $err_count++;
442 warn "query migrate failed: $err\n";
443 if ($err_count <= 5) {
444 usleep(1000000);
445 next;
446 }
447 die "too many query migrate failures - aborting\n";
448 }
985a5f48
AD
449
450 if ($stat->{status} =~ m/^(setup)$/im) {
451 sleep(1);
452 next;
453 }
454
5a7835f5 455 if ($stat->{status} =~ m/^(active|completed|failed|cancelled)$/im) {
d68afb26 456 $merr = undef;
b0b756c1 457 $err_count = 0;
5a7835f5 458 if ($stat->{status} eq 'completed') {
1e3baf05
DM
459 my $delay = time() - $start;
460 if ($delay > 0) {
461 my $mbps = sprintf "%.2f", $conf->{memory}/$delay;
135007c0
AD
462 my $downtime = $stat->{downtime} || 0;
463 $self->log('info', "migration speed: $mbps MB/s - downtime $downtime ms");
1e3baf05
DM
464 }
465 }
f5eb281a 466
5a7835f5 467 if ($stat->{status} eq 'failed' || $stat->{status} eq 'cancelled') {
1e3baf05
DM
468 die "aborting\n"
469 }
470
a05b47a8
DM
471 if ($stat->{status} ne 'active') {
472 $self->log('info', "migration status: $stat->{status}");
473 last;
474 }
475
476 if ($stat->{ram}->{transferred} ne $lstat) {
477 my $trans = $stat->{ram}->{transferred} || 0;
478 my $rem = $stat->{ram}->{remaining} || 0;
479 my $total = $stat->{ram}->{total} || 0;
e18b0b99
AD
480 my $xbzrlecachesize = $stat->{"xbzrle-cache"}->{"cache-size"} || 0;
481 my $xbzrlebytes = $stat->{"xbzrle-cache"}->{"bytes"} || 0;
482 my $xbzrlepages = $stat->{"xbzrle-cache"}->{"pages"} || 0;
483 my $xbzrlecachemiss = $stat->{"xbzrle-cache"}->{"cache-miss"} || 0;
484 my $xbzrleoverflow = $stat->{"xbzrle-cache"}->{"overflow"} || 0;
e52bd94c 485 #reduce sleep if remainig memory if lower than the everage transfert
94235c59 486 $usleep = 300000 if $avglstat && $rem < $avglstat;
a05b47a8
DM
487
488 $self->log('info', "migration status: $stat->{status} (transferred ${trans}, " .
0302101c 489 "remaining ${rem}), total ${total})");
e18b0b99 490
2e787b18
SP
491 if (${xbzrlecachesize}) {
492 $self->log('info', "migration xbzrle cachesize: ${xbzrlecachesize} transferred ${xbzrlebytes} pages ${xbzrlepages} cachemiss ${xbzrlecachemiss} overflow ${xbzrleoverflow}");
493 }
494
865ef132
SP
495 if (($lastrem && $rem > $lastrem ) || ($rem == 0)) {
496 $downtimecounter++;
497 }
498 $lastrem = $rem;
499
500 if ($downtimecounter > 5) {
501 $downtimecounter = 0;
502 $migrate_downtime *= 2;
503 $self->log('info', "migrate_set_downtime: $migrate_downtime");
504 eval {
505 PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate_set_downtime", value => int($migrate_downtime*100)/100);
506 };
507 $self->log('info', "migrate_set_downtime error: $@") if $@;
508 }
509
a05b47a8
DM
510 }
511
865ef132 512
a05b47a8 513 $lstat = $stat->{ram}->{transferred};
e52bd94c 514
1e3baf05 515 } else {
d68afb26 516 die $merr if $merr;
5a7835f5 517 die "unable to parse migration status '$stat->{status}' - aborting\n";
1e3baf05 518 }
a05b47a8 519 }
1e3baf05 520}
16e903f2 521
c04b5b04
AD
522sub phase2_cleanup {
523 my ($self, $vmid, $err) = @_;
524
af30308f
DM
525 return if !$self->{errors};
526 $self->{phase2errors} = 1;
527
c04b5b04
AD
528 $self->log('info', "aborting phase 2 - cleanup resources");
529
19168b91
SP
530 $self->log('info', "migrate_cancel");
531 eval {
532 PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate_cancel");
533 };
534 $self->log('info', "migrate_cancel error: $@") if $@;
535
c04b5b04
AD
536 my $conf = $self->{vmconf};
537 delete $conf->{lock};
538 eval { PVE::QemuServer::update_config_nolock($vmid, $conf, 1) };
539 if (my $err = $@) {
540 $self->log('err', $err);
541 }
542
af30308f
DM
543 # cleanup ressources on target host
544 my $nodename = PVE::INotify::nodename();
545
546 my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
547 eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
548 if (my $err = $@) {
549 $self->log('err', $err);
550 $self->{errors} = 1;
551 }
c04b5b04
AD
552}
553
16e903f2
DM
554sub phase3 {
555 my ($self, $vmid) = @_;
f5eb281a 556
16e903f2 557 my $volids = $self->{volumes};
af30308f 558 return if $self->{phase2errors};
16e903f2
DM
559
560 # destroy local copies
561 foreach my $volid (@$volids) {
562 eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
563 if (my $err = $@) {
564 $self->log('err', "removing local copy of '$volid' failed - $err");
565 $self->{errors} = 1;
566 last if $err =~ /^interrupted by signal$/;
567 }
568 }
16e903f2
DM
569}
570
571sub phase3_cleanup {
572 my ($self, $vmid, $err) = @_;
573
574 my $conf = $self->{vmconf};
af30308f 575 return if $self->{phase2errors};
16e903f2 576
b8d20802
AD
577 # move config to remote node
578 my $conffile = PVE::QemuServer::config_file($vmid);
579 my $newconffile = PVE::QemuServer::config_file($vmid, $self->{node});
580
581 die "Failed to move config to node '$self->{node}' - rename failed: $!\n"
582 if !rename($conffile, $newconffile);
583
5bc1e039
SP
584 if ($self->{livemigration}) {
585 # now that config file is move, we can resume vm on target if livemigrate
b67900f1 586 my $cmd = [@{$self->{rem_ssh}}, 'qm', 'resume', $vmid, '--skiplock'];
d5f315fd
AD
587 eval{ PVE::Tools::run_command($cmd, outfunc => sub {},
588 errfunc => sub {
589 my $line = shift;
590 $self->log('err', $line);
591 });
592 };
f5eb281a 593 if (my $err = $@) {
b67900f1
AD
594 $self->log('err', $err);
595 $self->{errors} = 1;
596 }
597 }
598
fd8469f7
AD
599 eval {
600
601 my $timer = 0;
602 if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running}) {
603 $self->log('info', "Waiting for spice server migration");
604 while (1) {
605 my $res = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'query-spice');
606 last if int($res->{'migrated'}) == 1;
607 last if $timer > 50;
608 $timer ++;
609 usleep(200000);
610 }
611 }
612 };
95a4b4a9 613
16e903f2
DM
614 # always stop local VM
615 eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); };
616 if (my $err = $@) {
617 $self->log('err', "stopping vm failed - $err");
618 $self->{errors} = 1;
619 }
620
97439670
DM
621 if ($self->{tunnel}) {
622 eval { finish_tunnel($self, $self->{tunnel}); };
623 if (my $err = $@) {
624 $self->log('err', $err);
625 $self->{errors} = 1;
626 }
627 }
628
16e903f2
DM
629 # always deactivate volumes - avoid lvm LVs to be active on several nodes
630 eval {
631 my $vollist = PVE::QemuServer::get_vm_volumes($conf);
632 PVE::Storage::deactivate_volumes($self->{storecfg}, $vollist);
633 };
634 if (my $err = $@) {
635 $self->log('err', $err);
636 $self->{errors} = 1;
637 }
638
639 # clear migrate lock
640 my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
641 $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
642}
643
644sub final_cleanup {
645 my ($self, $vmid) = @_;
646
647 # nothing to do
648}
649
6501;