[pve-guest-common.git] / src / PVE / AbstractMigrate.pm

package PVE::AbstractMigrate;

use strict;
use warnings;

use JSON;
use POSIX qw(strftime);

use PVE::Cluster;
use PVE::DataCenterConfig;
use PVE::ReplicationState;
use PVE::SSHInfo;
use PVE::Tools;

my $msg2text = sub {
    my ($level, $msg) = @_;

    chomp $msg;
    return '' if !$msg;

    my $res = '';

    my $tstr = strftime("%F %H:%M:%S", localtime);

    foreach my $line (split (/\n/, $msg)) {
	if ($level eq 'err') {
	    $res .= "$tstr ERROR: $line\n";
	} else {
	    $res .= "$tstr $line\n";
	}
    }

    return $res;
};

sub log {
    my ($self, $level, $msg) = @_;

    chomp $msg;
    return if !$msg;

    print $msg2text->($level, $msg);
}

sub cmd {
    my ($self, $cmd, %param) = @_;

    my $logfunc = sub {
	my $line = shift;
	$self->log('info', $line);
    };

    $self->log('info', "# " . PVE::Tools::cmd2string($cmd));

    PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
}

my $run_command_quiet_full = sub {
    my ($self, $cmd, $logerr, %param) = @_;

    my $log = '';
    my $logfunc = sub {
	my $line = shift;
	$log .= &$msg2text('info', $line);;
    };

    eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
    if (my $err = $@) {
	$self->log('info', "# " . PVE::Tools::cmd2string($cmd));
	print $log;
	if ($logerr) {
	    $self->{errors} = 1;
	    $self->log('err', $err);
	} else {
	    die $err;
	}
    }
};

sub cmd_quiet {
    my ($self, $cmd, %param) = @_;
    return &$run_command_quiet_full($self, $cmd, 0, %param);
}

sub cmd_logerr {
    my ($self, $cmd, %param) = @_;
    return &$run_command_quiet_full($self, $cmd, 1, %param);
}

my $eval_int = sub {
    my ($self, $func, @param) = @_;

    eval {
	local $SIG{INT} =
	    local $SIG{TERM} =
	    local $SIG{QUIT} =
	    local $SIG{HUP} =
	    local $SIG{PIPE} = sub {
		$self->{delayed_interrupt} = 0;
		die "interrupted by signal\n";
	};

	my $di = $self->{delayed_interrupt};
	$self->{delayed_interrupt} = 0;

	die "interrupted by signal\n" if $di;

	&$func($self, @param);
    };
};

# FIXME: nodeip is now unused
sub migrate {
    my ($class, $node, $nodeip, $vmid, $opts) = @_;

    $class = ref($class) || $class;

    my ($ssh_info, $rem_ssh);
    if (!$opts->{remote}) {
	my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');

	my $migration_network = $opts->{migration_network};
	if (!defined($migration_network)) {
	    $migration_network = $dc_conf->{migration}->{network};
	}
	$ssh_info = PVE::SSHInfo::get_ssh_info($node, $migration_network);
	$nodeip = $ssh_info->{ip};

	my $migration_type = 'secure';
	if (defined($opts->{migration_type})) {
	    $migration_type = $opts->{migration_type};
	} elsif (defined($dc_conf->{migration}->{type})) {
	    $migration_type = $dc_conf->{migration}->{type};
	}
	$opts->{migration_type} = $migration_type;
	$opts->{migration_network} = $migration_network;
	$rem_ssh = PVE::SSHInfo::ssh_info_to_command($ssh_info);
    }

    my $self = {
	delayed_interrupt => 0,
	opts => $opts,
	vmid => $vmid,
	node => $node,
	ssh_info => $ssh_info,
	nodeip => $nodeip,
	rem_ssh => $rem_ssh,
    };

    $self = bless $self, $class;

    my $starttime = time();

    local $SIG{INT} =
	local $SIG{TERM} =
	local $SIG{QUIT} =
	local $SIG{HUP} =
	local $SIG{PIPE} = sub {
	    $self->log('err', "received interrupt - delayed");
	    $self->{delayed_interrupt} = 1;
    };

    # lock container during migration
    eval { $self->lock_vm($self->{vmid}, sub {

	$self->{running} = 0;
	&$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
	die $@ if $@;

	if (defined($self->{opts}->{migration_network})) {
	    $self->log('info', "use dedicated network address for sending " .
	               "migration traffic ($self->{nodeip})");

	    # test if we can connect to new IP
	    my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
	    eval { $self->cmd_quiet($cmd); };
	    die "Can't connect to destination address ($self->{nodeip}) using " .
	        "public key authentication\n" if $@;
	}

	&$eval_int($self, sub { $self->phase1($self->{vmid}); });
	my $err = $@;
	if ($err) {
	    $self->log('err', $err);
	    eval { $self->phase1_cleanup($self->{vmid}, $err); };
	    if (my $tmperr = $@) {
		$self->log('err', $tmperr);
	    }
	    eval { $self->final_cleanup($self->{vmid}); };
	    if (my $tmperr = $@) {
		$self->log('err', $tmperr);
	    }
	    die $err;
	}

	# vm is now owned by other node
	# Note: there is no VM config file on the local node anymore

	if ($self->{running}) {

	    &$eval_int($self, sub { $self->phase2($self->{vmid}); });
	    my $phase2err = $@;
	    if ($phase2err) {
		$self->{errors} = 1;
		$self->log('err', "online migrate failure - $phase2err");
	    }
	    eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
	    if (my $err = $@) {
		$self->log('err', $err);
		$self->{errors} = 1;
	    }
	}

	# phase3 (finalize) 
	&$eval_int($self, sub { $self->phase3($self->{vmid}); });
	my $phase3err = $@;
	if ($phase3err) {
	    $self->log('err', $phase3err);
	    $self->{errors} = 1;
	}
	eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
	if (my $err = $@) {
	    $self->log('err', $err);
	    $self->{errors} = 1;
	}
	eval { $self->final_cleanup($self->{vmid}); };
	if (my $err = $@) {
	    $self->log('err', $err);
	    $self->{errors} = 1;
	}
    })};

    my $err = $@;

    my $delay = time() - $starttime;
    my $mins = int($delay/60);
    my $secs = $delay - $mins*60;
    my $hours =  int($mins/60);
    $mins = $mins - $hours*60;

    my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;

    if ($err) {
	$self->log('err', "migration aborted (duration $duration): $err");
	die "migration aborted\n";
    }

    if ($self->{errors}) {
	$self->log('err', "migration finished with problems (duration $duration)");
	die "migration problems\n"
    }

    $self->log('info', "migration finished successfully (duration $duration)");
}

sub lock_vm {
    my ($self, $vmid, $code, @param) = @_;

    die "abstract method - implement me";
}

sub prepare {
    my ($self, $vmid) = @_;

    die "abstract method - implement me";

    # return $running;
}

# transfer all data and move VM config files
sub phase1 {
    my ($self, $vmid) = @_;
    die "abstract method - implement me";
}

# only called if there are errors in phase1
sub phase1_cleanup {
    my ($self, $vmid, $err) = @_;
    die "abstract method - implement me";
}

# only called when VM is running and phase1 was successful
sub phase2 {
    my ($self, $vmid) = @_;
    die "abstract method - implement me";
}

# only called when VM is running and phase1 was successful
sub phase2_cleanup {
    my ($self, $vmid, $err) = @_;
};

#  only called when phase1 was successful
sub phase3 {
    my ($self, $vmid) = @_;
}

#  only called when phase1 was successful
sub phase3_cleanup {
    my ($self, $vmid, $err) = @_;
}

# final cleanup - always called
sub final_cleanup {
    my ($self, $vmid) = @_;
    die "abstract method - implement me";
}

# transfer replication state helper - call this before moving the guest config
# - just log the error is something fails
sub transfer_replication_state {
    my ($self) = @_;

    my $local_node = PVE::INotify::nodename();

    eval {
	my $stateobj = PVE::ReplicationState::read_state();
	my $vmstate = PVE::ReplicationState::extract_vmid_tranfer_state($stateobj, $self->{vmid}, $self->{node}, $local_node);
	# This have to be quoted when it run it over ssh.
	my $state = PVE::Tools::shellquote(encode_json($vmstate));
	my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $self->{vmid}, $state];
	$self->cmd($cmd);
    };
    if (my $err = $@) {
	$self->log('err', "transfer replication state failed - $err");
	$self->{errors} = 1;
    }
}

# switch replication job target - call this after moving the guest config
# - does nothing if there is no relevant replication job
# - just log the error is something fails
sub switch_replication_job_target {
    my ($self) = @_;

    my $local_node = PVE::INotify::nodename();

    eval { PVE::ReplicationConfig::switch_replication_job_target($self->{vmid}, $self->{node}, $local_node); };
    if (my $err = $@) {
	$self->log('err', "switch replication job target failed - $err");
	$self->{errors} = 1;
    }
}

# merges local limit '$bwlimit' and a possible remote limit
sub get_bwlimit {
    my ($self, $source, $target) = @_;

    my $local_sids = $self->{opts}->{remote} ? [$source] : [$source, $target];

    my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', $local_sids, $self->{opts}->{bwlimit});

    if ($self->{opts}->{remote}) {
	my $bwlimit_opts = {
	    operation => 'migration',
	    storages => [$target],
	    bwlimit => $self->{opts}->{bwlimit},
	};
	my $remote_bwlimit = PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'bwlimit', $bwlimit_opts);
	if ($remote_bwlimit && $remote_bwlimit->{bwlimit}) {
	    $remote_bwlimit = $remote_bwlimit->{bwlimit};

	    $bwlimit = $remote_bwlimit
		if (!$bwlimit || $bwlimit > $remote_bwlimit);
	}
    }

    return $bwlimit;
}

1;
Commit	Line	Data
58a3c91c FG	1	package PVE::AbstractMigrate;
	2
	3	use strict;
	4	use warnings;
790f7f1f	5
1c960710	6	use JSON;
790f7f1f TL	7	use POSIX qw(strftime);
790f7f1f TL	8
9591475a	9	use PVE::Cluster;
96c08a9d	10	use PVE::DataCenterConfig;
1c960710	11	use PVE::ReplicationState;
0c85474f	12	use PVE::SSHInfo;
790f7f1f	13	use PVE::Tools;
58a3c91c FG	14
	15	my $msg2text = sub {
	16	my ($level, $msg) = @_;
	17
	18	chomp $msg;
58a3c91c FG	19	return '' if !$msg;
	20
	21	my $res = '';
	22
9ed0fe88	23	my $tstr = strftime("%F %H:%M:%S", localtime);
58a3c91c FG	24
	25	foreach my $line (split (/\n/, $msg)) {
	26	if ($level eq 'err') {
	27	$res .= "$tstr ERROR: $line\n";
	28	} else {
	29	$res .= "$tstr $line\n";
	30	}
	31	}
	32
	33	return $res;
	34	};
	35
	36	sub log {
	37	my ($self, $level, $msg) = @_;
	38
	39	chomp $msg;
58a3c91c FG	40	return if !$msg;
58a3c91c FG	41
2b062714	42	print $msg2text->($level, $msg);
58a3c91c FG	43	}
	44
	45	sub cmd {
	46	my ($self, $cmd, %param) = @_;
	47
	48	my $logfunc = sub {
	49	my $line = shift;
	50	$self->log('info', $line);
	51	};
	52
	53	$self->log('info', "# " . PVE::Tools::cmd2string($cmd));
	54
	55	PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
	56	}
	57
	58	my $run_command_quiet_full = sub {
	59	my ($self, $cmd, $logerr, %param) = @_;
	60
	61	my $log = '';
	62	my $logfunc = sub {
	63	my $line = shift;
	64	$log .= &$msg2text('info', $line);;
	65	};
	66
	67	eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
	68	if (my $err = $@) {
	69	$self->log('info', "# " . PVE::Tools::cmd2string($cmd));
	70	print $log;
	71	if ($logerr) {
	72	$self->{errors} = 1;
	73	$self->log('err', $err);
	74	} else {
	75	die $err;
	76	}
	77	}
	78	};
	79
	80	sub cmd_quiet {
	81	my ($self, $cmd, %param) = @_;
	82	return &$run_command_quiet_full($self, $cmd, 0, %param);
	83	}
	84
	85	sub cmd_logerr {
	86	my ($self, $cmd, %param) = @_;
	87	return &$run_command_quiet_full($self, $cmd, 1, %param);
	88	}
	89
58a3c91c FG	90	my $eval_int = sub {
	91	my ($self, $func, @param) = @_;
	92
	93	eval {
ee966a3f TL	94	local $SIG{INT} =
	95	local $SIG{TERM} =
	96	local $SIG{QUIT} =
	97	local $SIG{HUP} =
	98	local $SIG{PIPE} = sub {
	99	$self->{delayed_interrupt} = 0;
	100	die "interrupted by signal\n";
58a3c91c FG	101	};
	102
	103	my $di = $self->{delayed_interrupt};
	104	$self->{delayed_interrupt} = 0;
	105
	106	die "interrupted by signal\n" if $di;
	107
	108	&$func($self, @param);
	109	};
	110	};
	111
9591475a	112	# FIXME: nodeip is now unused
58a3c91c FG	113	sub migrate {
	114	my ($class, $node, $nodeip, $vmid, $opts) = @_;
	115
	116	$class = ref($class) \|\| $class;
	117
42a84dc9 FG	118	my ($ssh_info, $rem_ssh);
	119	if (!$opts->{remote}) {
	120	my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
40318e97	121
42a84dc9 FG	122	my $migration_network = $opts->{migration_network};
	123	if (!defined($migration_network)) {
	124	$migration_network = $dc_conf->{migration}->{network};
	125	}
	126	$ssh_info = PVE::SSHInfo::get_ssh_info($node, $migration_network);
	127	$nodeip = $ssh_info->{ip};
	128
	129	my $migration_type = 'secure';
	130	if (defined($opts->{migration_type})) {
	131	$migration_type = $opts->{migration_type};
	132	} elsif (defined($dc_conf->{migration}->{type})) {
	133	$migration_type = $dc_conf->{migration}->{type};
	134	}
	135	$opts->{migration_type} = $migration_type;
	136	$opts->{migration_network} = $migration_network;
	137	$rem_ssh = PVE::SSHInfo::ssh_info_to_command($ssh_info);
40318e97	138	}
40318e97	139
58a3c91c FG	140	my $self = {
	141	delayed_interrupt => 0,
	142	opts => $opts,
	143	vmid => $vmid,
	144	node => $node,
9591475a	145	ssh_info => $ssh_info,
58a3c91c	146	nodeip => $nodeip,
42a84dc9	147	rem_ssh => $rem_ssh,
58a3c91c FG	148	};
	149
	150	$self = bless $self, $class;
	151
	152	my $starttime = time();
	153
ee966a3f TL	154	local $SIG{INT} =
	155	local $SIG{TERM} =
	156	local $SIG{QUIT} =
	157	local $SIG{HUP} =
	158	local $SIG{PIPE} = sub {
	159	$self->log('err', "received interrupt - delayed");
	160	$self->{delayed_interrupt} = 1;
58a3c91c FG	161	};
58a3c91c FG	162
58a3c91c FG	163	# lock container during migration
	164	eval { $self->lock_vm($self->{vmid}, sub {
	165
	166	$self->{running} = 0;
	167	&$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
	168	die $@ if $@;
	169
42a84dc9	170	if (defined($self->{opts}->{migration_network})) {
58a3c91c FG	171	$self->log('info', "use dedicated network address for sending " .
	172	"migration traffic ($self->{nodeip})");
	173
	174	# test if we can connect to new IP
	175	my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
	176	eval { $self->cmd_quiet($cmd); };
	177	die "Can't connect to destination address ($self->{nodeip}) using " .
	178	"public key authentication\n" if $@;
	179	}
	180
	181	&$eval_int($self, sub { $self->phase1($self->{vmid}); });
	182	my $err = $@;
	183	if ($err) {
	184	$self->log('err', $err);
	185	eval { $self->phase1_cleanup($self->{vmid}, $err); };
	186	if (my $tmperr = $@) {
	187	$self->log('err', $tmperr);
	188	}
	189	eval { $self->final_cleanup($self->{vmid}); };
	190	if (my $tmperr = $@) {
	191	$self->log('err', $tmperr);
	192	}
	193	die $err;
	194	}
	195
	196	# vm is now owned by other node
	197	# Note: there is no VM config file on the local node anymore
	198
	199	if ($self->{running}) {
	200
	201	&$eval_int($self, sub { $self->phase2($self->{vmid}); });
	202	my $phase2err = $@;
	203	if ($phase2err) {
	204	$self->{errors} = 1;
	205	$self->log('err', "online migrate failure - $phase2err");
	206	}
	207	eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
	208	if (my $err = $@) {
	209	$self->log('err', $err);
	210	$self->{errors} = 1;
	211	}
	212	}
	213
	214	# phase3 (finalize)
	215	&$eval_int($self, sub { $self->phase3($self->{vmid}); });
	216	my $phase3err = $@;
	217	if ($phase3err) {
	218	$self->log('err', $phase3err);
	219	$self->{errors} = 1;
	220	}
	221	eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
	222	if (my $err = $@) {
	223	$self->log('err', $err);
	224	$self->{errors} = 1;
	225	}
	226	eval { $self->final_cleanup($self->{vmid}); };
	227	if (my $err = $@) {
	228	$self->log('err', $err);
	229	$self->{errors} = 1;
	230	}
	231	})};
	232
	233	my $err = $@;
	234
235	my $delay = time() - $starttime;
236	my $mins = int($delay/60);
237	my $secs = $delay - $mins*60;
238	my $hours = int($mins/60);
239	$mins = $mins - $hours*60;
240
241	my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;
242
243	if ($err) {
244	$self->log('err', "migration aborted (duration $duration): $err");
245	die "migration aborted\n";
246	}
247
248	if ($self->{errors}) {
249	$self->log('err', "migration finished with problems (duration $duration)");
250	die "migration problems\n"
251	}
252
253	$self->log('info', "migration finished successfully (duration $duration)");
254	}
255
256	sub lock_vm {
257	my ($self, $vmid, $code, @param) = @_;
258
259	die "abstract method - implement me";
260	}
261
262	sub prepare {
263	my ($self, $vmid) = @_;
264
265	die "abstract method - implement me";
266
267	# return $running;
268	}
269
270	# transfer all data and move VM config files
271	sub phase1 {
272	my ($self, $vmid) = @_;
273	die "abstract method - implement me";
274	}
275
276	# only called if there are errors in phase1
277	sub phase1_cleanup {
278	my ($self, $vmid, $err) = @_;
279	die "abstract method - implement me";
280	}
281
282	# only called when VM is running and phase1 was successful
283	sub phase2 {
284	my ($self, $vmid) = @_;
285	die "abstract method - implement me";
286	}
287
288	# only called when VM is running and phase1 was successful
289	sub phase2_cleanup {
290	my ($self, $vmid, $err) = @_;
291	};
292
293	# only called when phase1 was successful
294	sub phase3 {
295	my ($self, $vmid) = @_;
296	}
297
298	# only called when phase1 was successful
299	sub phase3_cleanup {
300	my ($self, $vmid, $err) = @_;
301	}
302
303	# final cleanup - always called
304	sub final_cleanup {
305	my ($self, $vmid) = @_;
306	die "abstract method - implement me";
307	}
308
1c960710 DM	309	# transfer replication state helper - call this before moving the guest config
	310	# - just log the error is something fails
	311	sub transfer_replication_state {
	312	my ($self) = @_;
	313
	314	my $local_node = PVE::INotify::nodename();
	315
	316	eval {
	317	my $stateobj = PVE::ReplicationState::read_state();
	318	my $vmstate = PVE::ReplicationState::extract_vmid_tranfer_state($stateobj, $self->{vmid}, $self->{node}, $local_node);
	319	# This have to be quoted when it run it over ssh.
	320	my $state = PVE::Tools::shellquote(encode_json($vmstate));
	321	my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $self->{vmid}, $state];
	322	$self->cmd($cmd);
	323	};
	324	if (my $err = $@) {
	325	$self->log('err', "transfer replication state failed - $err");
	326	$self->{errors} = 1;
	327	}
	328	}
	329
	330	# switch replication job target - call this after moving the guest config
	331	# - does nothing if there is no relevant replication job
	332	# - just log the error is something fails
	333	sub switch_replication_job_target {
	334	my ($self) = @_;
	335
	336	my $local_node = PVE::INotify::nodename();
	337
	338	eval { PVE::ReplicationConfig::switch_replication_job_target($self->{vmid}, $self->{node}, $local_node); };
	339	if (my $err = $@) {
	340	$self->log('err', "switch replication job target failed - $err");
	341	$self->{errors} = 1;
	342	}
	343	}
	344
74c26370 FG	345	# merges local limit '$bwlimit' and a possible remote limit
	346	sub get_bwlimit {
	347	my ($self, $source, $target) = @_;
	348
	349	my $local_sids = $self->{opts}->{remote} ? [$source] : [$source, $target];
	350
	351	my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', $local_sids, $self->{opts}->{bwlimit});
	352
	353	if ($self->{opts}->{remote}) {
	354	my $bwlimit_opts = {
	355	operation => 'migration',
	356	storages => [$target],
	357	bwlimit => $self->{opts}->{bwlimit},
	358	};
	359	my $remote_bwlimit = PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'bwlimit', $bwlimit_opts);
	360	if ($remote_bwlimit && $remote_bwlimit->{bwlimit}) {
	361	$remote_bwlimit = $remote_bwlimit->{bwlimit};
	362
	363	$bwlimit = $remote_bwlimit
	364	if (!$bwlimit \|\| $bwlimit > $remote_bwlimit);
	365	}
	366	}
	367
	368	return $bwlimit;
	369	}
	370
58a3c91c	371	1;