[pve-guest-common.git] / PVE / AbstractMigrate.pm

package PVE::AbstractMigrate;

use strict;
use warnings;
use POSIX qw(strftime);
use JSON;
use PVE::Tools;
use PVE::Cluster;
use PVE::ReplicationState;

my $msg2text = sub {
    my ($level, $msg) = @_;

    chomp $msg;

    return '' if !$msg;

    my $res = '';

    my $tstr = strftime("%F %H:%M:%S", localtime);

    foreach my $line (split (/\n/, $msg)) {
	if ($level eq 'err') {
	    $res .= "$tstr ERROR: $line\n";
	} else {
	    $res .= "$tstr $line\n";
	}
    }

    return $res;
};

sub log {
    my ($self, $level, $msg) = @_;

    chomp $msg;

    return if !$msg;

    print &$msg2text($level, $msg);
}

sub cmd {
    my ($self, $cmd, %param) = @_;

    my $logfunc = sub {
	my $line = shift;
	$self->log('info', $line);
    };

    $self->log('info', "# " . PVE::Tools::cmd2string($cmd));

    PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
}

my $run_command_quiet_full = sub {
    my ($self, $cmd, $logerr, %param) = @_;

    my $log = '';
    my $logfunc = sub {
	my $line = shift;
	$log .= &$msg2text('info', $line);;
    };

    eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
    if (my $err = $@) {
	$self->log('info', "# " . PVE::Tools::cmd2string($cmd));
	print $log;
	if ($logerr) {
	    $self->{errors} = 1;
	    $self->log('err', $err);
	} else {
	    die $err;
	}
    }
};

sub cmd_quiet {
    my ($self, $cmd, %param) = @_;
    return &$run_command_quiet_full($self, $cmd, 0, %param);
}

sub cmd_logerr {
    my ($self, $cmd, %param) = @_;
    return &$run_command_quiet_full($self, $cmd, 1, %param);
}

my $eval_int = sub {
    my ($self, $func, @param) = @_;

    eval {
	local $SIG{INT} =
	    local $SIG{TERM} =
	    local $SIG{QUIT} =
	    local $SIG{HUP} =
	    local $SIG{PIPE} = sub {
		$self->{delayed_interrupt} = 0;
		die "interrupted by signal\n";
	};

	my $di = $self->{delayed_interrupt};
	$self->{delayed_interrupt} = 0;

	die "interrupted by signal\n" if $di;

	&$func($self, @param);
    };
};

# FIXME: nodeip is now unused
sub migrate {
    my ($class, $node, $nodeip, $vmid, $opts) = @_;

    $class = ref($class) || $class;

    my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');

    my $migration_network = $opts->{migration_network};
    if (!defined($migration_network)) {
	$migration_network = $dc_conf->{migration}->{network};
    }
    my $ssh_info = PVE::Cluster::get_ssh_info($node, $migration_network);
    $nodeip = $ssh_info->{ip};

    my $migration_type = 'secure';
    if (defined($opts->{migration_type})) {
	$migration_type = $opts->{migration_type};
    } elsif (defined($dc_conf->{migration}->{type})) {
        $migration_type = $dc_conf->{migration}->{type};
    }
    $opts->{migration_type} = $migration_type;

    my $self = {
	delayed_interrupt => 0,
	opts => $opts,
	vmid => $vmid,
	node => $node,
	ssh_info => $ssh_info,
	nodeip => $nodeip,
	rem_ssh => PVE::Cluster::ssh_info_to_command($ssh_info)
    };

    $self = bless $self, $class;

    my $starttime = time();

    local $SIG{INT} =
	local $SIG{TERM} =
	local $SIG{QUIT} =
	local $SIG{HUP} =
	local $SIG{PIPE} = sub {
	    $self->log('err', "received interrupt - delayed");
	    $self->{delayed_interrupt} = 1;
    };

    # lock container during migration
    eval { $self->lock_vm($self->{vmid}, sub {

	$self->{running} = 0;
	&$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
	die $@ if $@;

	if (defined($migration_network)) {
	    $self->log('info', "use dedicated network address for sending " .
	               "migration traffic ($self->{nodeip})");

	    # test if we can connect to new IP
	    my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
	    eval { $self->cmd_quiet($cmd); };
	    die "Can't connect to destination address ($self->{nodeip}) using " .
	        "public key authentication\n" if $@;
	}

	&$eval_int($self, sub { $self->phase1($self->{vmid}); });
	my $err = $@;
	if ($err) {
	    $self->log('err', $err);
	    eval { $self->phase1_cleanup($self->{vmid}, $err); };
	    if (my $tmperr = $@) {
		$self->log('err', $tmperr);
	    }
	    eval { $self->final_cleanup($self->{vmid}); };
	    if (my $tmperr = $@) {
		$self->log('err', $tmperr);
	    }
	    die $err;
	}

	# vm is now owned by other node
	# Note: there is no VM config file on the local node anymore

	if ($self->{running}) {

	    &$eval_int($self, sub { $self->phase2($self->{vmid}); });
	    my $phase2err = $@;
	    if ($phase2err) {
		$self->{errors} = 1;
		$self->log('err', "online migrate failure - $phase2err");
	    }
	    eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
	    if (my $err = $@) {
		$self->log('err', $err);
		$self->{errors} = 1;
	    }
	}

	# phase3 (finalize) 
	&$eval_int($self, sub { $self->phase3($self->{vmid}); });
	my $phase3err = $@;
	if ($phase3err) {
	    $self->log('err', $phase3err);
	    $self->{errors} = 1;
	}
	eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
	if (my $err = $@) {
	    $self->log('err', $err);
	    $self->{errors} = 1;
	}
	eval { $self->final_cleanup($self->{vmid}); };
	if (my $err = $@) {
	    $self->log('err', $err);
	    $self->{errors} = 1;
	}
    })};

    my $err = $@;

    my $delay = time() - $starttime;
    my $mins = int($delay/60);
    my $secs = $delay - $mins*60;
    my $hours =  int($mins/60);
    $mins = $mins - $hours*60;

    my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;

    if ($err) {
	$self->log('err', "migration aborted (duration $duration): $err");
	die "migration aborted\n";
    }

    if ($self->{errors}) {
	$self->log('err', "migration finished with problems (duration $duration)");
	die "migration problems\n"
    }

    $self->log('info', "migration finished successfully (duration $duration)");
}

sub lock_vm {
    my ($self, $vmid, $code, @param) = @_;

    die "abstract method - implement me";
}

sub prepare {
    my ($self, $vmid) = @_;

    die "abstract method - implement me";

    # return $running;
}

# transfer all data and move VM config files
sub phase1 {
    my ($self, $vmid) = @_;
    die "abstract method - implement me";
}

# only called if there are errors in phase1
sub phase1_cleanup {
    my ($self, $vmid, $err) = @_;
    die "abstract method - implement me";
}

# only called when VM is running and phase1 was successful
sub phase2 {
    my ($self, $vmid) = @_;
    die "abstract method - implement me";
}

# only called when VM is running and phase1 was successful
sub phase2_cleanup {
    my ($self, $vmid, $err) = @_;
};

#  only called when phase1 was successful
sub phase3 {
    my ($self, $vmid) = @_;
}

#  only called when phase1 was successful
sub phase3_cleanup {
    my ($self, $vmid, $err) = @_;
}

# final cleanup - always called
sub final_cleanup {
    my ($self, $vmid) = @_;
    die "abstract method - implement me";
}

# transfer replication state helper - call this before moving the guest config
# - just log the error is something fails
sub transfer_replication_state {
    my ($self) = @_;

    my $local_node = PVE::INotify::nodename();

    eval {
	my $stateobj = PVE::ReplicationState::read_state();
	my $vmstate = PVE::ReplicationState::extract_vmid_tranfer_state($stateobj, $self->{vmid}, $self->{node}, $local_node);
	# This have to be quoted when it run it over ssh.
	my $state = PVE::Tools::shellquote(encode_json($vmstate));
	my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $self->{vmid}, $state];
	$self->cmd($cmd);
    };
    if (my $err = $@) {
	$self->log('err', "transfer replication state failed - $err");
	$self->{errors} = 1;
    }
}

# switch replication job target - call this after moving the guest config
# - does nothing if there is no relevant replication job
# - just log the error is something fails
sub switch_replication_job_target {
    my ($self) = @_;

    my $local_node = PVE::INotify::nodename();

    eval { PVE::ReplicationConfig::switch_replication_job_target($self->{vmid}, $self->{node}, $local_node); };
    if (my $err = $@) {
	$self->log('err', "switch replication job target failed - $err");
	$self->{errors} = 1;
    }
}

1;
Commit	Line	Data
58a3c91c FG	1	package PVE::AbstractMigrate;
	2
	3	use strict;
	4	use warnings;
	5	use POSIX qw(strftime);
1c960710	6	use JSON;
58a3c91c	7	use PVE::Tools;
9591475a	8	use PVE::Cluster;
1c960710	9	use PVE::ReplicationState;
58a3c91c FG	10
	11	my $msg2text = sub {
	12	my ($level, $msg) = @_;
	13
	14	chomp $msg;
	15
	16	return '' if !$msg;
	17
	18	my $res = '';
	19
9ed0fe88	20	my $tstr = strftime("%F %H:%M:%S", localtime);
58a3c91c FG	21
	22	foreach my $line (split (/\n/, $msg)) {
	23	if ($level eq 'err') {
	24	$res .= "$tstr ERROR: $line\n";
	25	} else {
	26	$res .= "$tstr $line\n";
	27	}
	28	}
	29
	30	return $res;
	31	};
	32
	33	sub log {
	34	my ($self, $level, $msg) = @_;
	35
	36	chomp $msg;
	37
	38	return if !$msg;
	39
	40	print &$msg2text($level, $msg);
	41	}
	42
	43	sub cmd {
	44	my ($self, $cmd, %param) = @_;
	45
	46	my $logfunc = sub {
	47	my $line = shift;
	48	$self->log('info', $line);
	49	};
	50
	51	$self->log('info', "# " . PVE::Tools::cmd2string($cmd));
	52
	53	PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
	54	}
	55
	56	my $run_command_quiet_full = sub {
	57	my ($self, $cmd, $logerr, %param) = @_;
	58
	59	my $log = '';
	60	my $logfunc = sub {
	61	my $line = shift;
	62	$log .= &$msg2text('info', $line);;
	63	};
	64
	65	eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
	66	if (my $err = $@) {
	67	$self->log('info', "# " . PVE::Tools::cmd2string($cmd));
	68	print $log;
	69	if ($logerr) {
	70	$self->{errors} = 1;
	71	$self->log('err', $err);
	72	} else {
	73	die $err;
	74	}
	75	}
	76	};
	77
	78	sub cmd_quiet {
	79	my ($self, $cmd, %param) = @_;
	80	return &$run_command_quiet_full($self, $cmd, 0, %param);
	81	}
	82
	83	sub cmd_logerr {
	84	my ($self, $cmd, %param) = @_;
85	return &$run_command_quiet_full($self, $cmd, 1, %param);
86	}
87
58a3c91c FG	88	my $eval_int = sub {
	89	my ($self, $func, @param) = @_;
	90
	91	eval {
ee966a3f TL	92	local $SIG{INT} =
	93	local $SIG{TERM} =
	94	local $SIG{QUIT} =
	95	local $SIG{HUP} =
	96	local $SIG{PIPE} = sub {
	97	$self->{delayed_interrupt} = 0;
	98	die "interrupted by signal\n";
58a3c91c FG	99	};
	100
	101	my $di = $self->{delayed_interrupt};
	102	$self->{delayed_interrupt} = 0;
	103
	104	die "interrupted by signal\n" if $di;
	105
	106	&$func($self, @param);
	107	};
	108	};
	109
9591475a	110	# FIXME: nodeip is now unused
58a3c91c FG	111	sub migrate {
	112	my ($class, $node, $nodeip, $vmid, $opts) = @_;
	113
	114	$class = ref($class) \|\| $class;
	115
40318e97 WB	116	my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
40318e97 WB	117
9591475a WB	118	my $migration_network = $opts->{migration_network};
9591475a WB	119	if (!defined($migration_network)) {
9591475a WB	120	$migration_network = $dc_conf->{migration}->{network};
	121	}
	122	my $ssh_info = PVE::Cluster::get_ssh_info($node, $migration_network);
	123	$nodeip = $ssh_info->{ip};
	124
40318e97 WB	125	my $migration_type = 'secure';
	126	if (defined($opts->{migration_type})) {
	127	$migration_type = $opts->{migration_type};
	128	} elsif (defined($dc_conf->{migration}->{type})) {
	129	$migration_type = $dc_conf->{migration}->{type};
	130	}
	131	$opts->{migration_type} = $migration_type;
	132
58a3c91c FG	133	my $self = {
	134	delayed_interrupt => 0,
	135	opts => $opts,
	136	vmid => $vmid,
	137	node => $node,
9591475a	138	ssh_info => $ssh_info,
58a3c91c	139	nodeip => $nodeip,
9591475a	140	rem_ssh => PVE::Cluster::ssh_info_to_command($ssh_info)
58a3c91c FG	141	};
	142
	143	$self = bless $self, $class;
	144
	145	my $starttime = time();
	146
ee966a3f TL	147	local $SIG{INT} =
	148	local $SIG{TERM} =
	149	local $SIG{QUIT} =
	150	local $SIG{HUP} =
	151	local $SIG{PIPE} = sub {
	152	$self->log('err', "received interrupt - delayed");
	153	$self->{delayed_interrupt} = 1;
58a3c91c FG	154	};
58a3c91c FG	155
58a3c91c FG	156	# lock container during migration
	157	eval { $self->lock_vm($self->{vmid}, sub {
	158
	159	$self->{running} = 0;
	160	&$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
	161	die $@ if $@;
	162
9591475a	163	if (defined($migration_network)) {
58a3c91c FG	164	$self->log('info', "use dedicated network address for sending " .
	165	"migration traffic ($self->{nodeip})");
	166
	167	# test if we can connect to new IP
	168	my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
	169	eval { $self->cmd_quiet($cmd); };
	170	die "Can't connect to destination address ($self->{nodeip}) using " .
	171	"public key authentication\n" if $@;
	172	}
	173
	174	&$eval_int($self, sub { $self->phase1($self->{vmid}); });
	175	my $err = $@;
	176	if ($err) {
	177	$self->log('err', $err);
	178	eval { $self->phase1_cleanup($self->{vmid}, $err); };
	179	if (my $tmperr = $@) {
	180	$self->log('err', $tmperr);
	181	}
	182	eval { $self->final_cleanup($self->{vmid}); };
	183	if (my $tmperr = $@) {
	184	$self->log('err', $tmperr);
	185	}
	186	die $err;
	187	}
	188
	189	# vm is now owned by other node
	190	# Note: there is no VM config file on the local node anymore
	191
	192	if ($self->{running}) {
	193
	194	&$eval_int($self, sub { $self->phase2($self->{vmid}); });
	195	my $phase2err = $@;
	196	if ($phase2err) {
	197	$self->{errors} = 1;
	198	$self->log('err', "online migrate failure - $phase2err");
	199	}
	200	eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
	201	if (my $err = $@) {
	202	$self->log('err', $err);
	203	$self->{errors} = 1;
	204	}
	205	}
	206
	207	# phase3 (finalize)
	208	&$eval_int($self, sub { $self->phase3($self->{vmid}); });
	209	my $phase3err = $@;
	210	if ($phase3err) {
	211	$self->log('err', $phase3err);
	212	$self->{errors} = 1;
	213	}
	214	eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
	215	if (my $err = $@) {
	216	$self->log('err', $err);
	217	$self->{errors} = 1;
	218	}
	219	eval { $self->final_cleanup($self->{vmid}); };
	220	if (my $err = $@) {
	221	$self->log('err', $err);
	222	$self->{errors} = 1;
	223	}
	224	})};
	225
	226	my $err = $@;
	227
228	my $delay = time() - $starttime;
229	my $mins = int($delay/60);
230	my $secs = $delay - $mins*60;
231	my $hours = int($mins/60);
232	$mins = $mins - $hours*60;
233
234	my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;
235
236	if ($err) {
237	$self->log('err', "migration aborted (duration $duration): $err");
238	die "migration aborted\n";
239	}
240
241	if ($self->{errors}) {
242	$self->log('err', "migration finished with problems (duration $duration)");
243	die "migration problems\n"
244	}
245
246	$self->log('info', "migration finished successfully (duration $duration)");
247	}
248
249	sub lock_vm {
250	my ($self, $vmid, $code, @param) = @_;
251
252	die "abstract method - implement me";
253	}
254
255	sub prepare {
256	my ($self, $vmid) = @_;
257
258	die "abstract method - implement me";
259
260	# return $running;
261	}
262
263	# transfer all data and move VM config files
264	sub phase1 {
265	my ($self, $vmid) = @_;
266	die "abstract method - implement me";
267	}
268
269	# only called if there are errors in phase1
270	sub phase1_cleanup {
271	my ($self, $vmid, $err) = @_;
272	die "abstract method - implement me";
273	}
274
275	# only called when VM is running and phase1 was successful
276	sub phase2 {
277	my ($self, $vmid) = @_;
278	die "abstract method - implement me";
279	}
280
281	# only called when VM is running and phase1 was successful
282	sub phase2_cleanup {
283	my ($self, $vmid, $err) = @_;
284	};
285
286	# only called when phase1 was successful
287	sub phase3 {
288	my ($self, $vmid) = @_;
289	}
290
291	# only called when phase1 was successful
292	sub phase3_cleanup {
293	my ($self, $vmid, $err) = @_;
294	}
295
296	# final cleanup - always called
297	sub final_cleanup {
298	my ($self, $vmid) = @_;
299	die "abstract method - implement me";
300	}
301
1c960710 DM	302	# transfer replication state helper - call this before moving the guest config
	303	# - just log the error is something fails
	304	sub transfer_replication_state {
	305	my ($self) = @_;
	306
	307	my $local_node = PVE::INotify::nodename();
	308
	309	eval {
	310	my $stateobj = PVE::ReplicationState::read_state();
	311	my $vmstate = PVE::ReplicationState::extract_vmid_tranfer_state($stateobj, $self->{vmid}, $self->{node}, $local_node);
	312	# This have to be quoted when it run it over ssh.
	313	my $state = PVE::Tools::shellquote(encode_json($vmstate));
	314	my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $self->{vmid}, $state];
	315	$self->cmd($cmd);
	316	};
	317	if (my $err = $@) {
	318	$self->log('err', "transfer replication state failed - $err");
	319	$self->{errors} = 1;
	320	}
	321	}
	322
	323	# switch replication job target - call this after moving the guest config
	324	# - does nothing if there is no relevant replication job
	325	# - just log the error is something fails
	326	sub switch_replication_job_target {
	327	my ($self) = @_;
	328
	329	my $local_node = PVE::INotify::nodename();
	330
	331	eval { PVE::ReplicationConfig::switch_replication_job_target($self->{vmid}, $self->{node}, $local_node); };
	332	if (my $err = $@) {
	333	$self->log('err', "switch replication job target failed - $err");
	334	$self->{errors} = 1;
	335	}
	336	}
	337
58a3c91c	338	1;