[pve-guest-common.git] / PVE / AbstractMigrate.pm

package PVE::AbstractMigrate;

use strict;
use warnings;
use POSIX qw(strftime);
use JSON;
use PVE::Tools;
use PVE::Cluster;
use PVE::ReplicationState;

my $msg2text = sub {
    my ($level, $msg) = @_;

    chomp $msg;

    return '' if !$msg;

    my $res = '';

    my $tstr = strftime("%F %H:%M:%S", localtime);

    foreach my $line (split (/\n/, $msg)) {
	if ($level eq 'err') {
	    $res .= "$tstr ERROR: $line\n";
	} else {
	    $res .= "$tstr $line\n";
	}
    }

    return $res;
};

sub log {
    my ($self, $level, $msg) = @_;

    chomp $msg;

    return if !$msg;

    print &$msg2text($level, $msg);
}

sub cmd {
    my ($self, $cmd, %param) = @_;

    my $logfunc = sub {
	my $line = shift;
	$self->log('info', $line);
    };

    $self->log('info', "# " . PVE::Tools::cmd2string($cmd));

    PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
}

my $run_command_quiet_full = sub {
    my ($self, $cmd, $logerr, %param) = @_;

    my $log = '';
    my $logfunc = sub {
	my $line = shift;
	$log .= &$msg2text('info', $line);;
    };

    eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
    if (my $err = $@) {
	$self->log('info', "# " . PVE::Tools::cmd2string($cmd));
	print $log;
	if ($logerr) {
	    $self->{errors} = 1;
	    $self->log('err', $err);
	} else {
	    die $err;
	}
    }
};

sub cmd_quiet {
    my ($self, $cmd, %param) = @_;
    return &$run_command_quiet_full($self, $cmd, 0, %param);
}

sub cmd_logerr {
    my ($self, $cmd, %param) = @_;
    return &$run_command_quiet_full($self, $cmd, 1, %param);
}

my $eval_int = sub {
    my ($self, $func, @param) = @_;

    eval {
	local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
	    $self->{delayed_interrupt} = 0;
	    die "interrupted by signal\n";
	};
	local $SIG{PIPE} = sub {
	    $self->{delayed_interrupt} = 0;
	    die "interrupted by signal\n";
	};

	my $di = $self->{delayed_interrupt};
	$self->{delayed_interrupt} = 0;

	die "interrupted by signal\n" if $di;

	&$func($self, @param);
    };
};

my @ssh_opts = ('-o', 'BatchMode=yes');
my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);

# FIXME: nodeip is now unused
sub migrate {
    my ($class, $node, $nodeip, $vmid, $opts) = @_;

    $class = ref($class) || $class;

    my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');

    my $migration_network = $opts->{migration_network};
    if (!defined($migration_network)) {
	$migration_network = $dc_conf->{migration}->{network};
    }
    my $ssh_info = PVE::Cluster::get_ssh_info($node, $migration_network);
    $nodeip = $ssh_info->{ip};

    my $migration_type = 'secure';
    if (defined($opts->{migration_type})) {
	$migration_type = $opts->{migration_type};
    } elsif (defined($dc_conf->{migration}->{type})) {
        $migration_type = $dc_conf->{migration}->{type};
    }
    $opts->{migration_type} = $migration_type;

    my $self = {
	delayed_interrupt => 0,
	opts => $opts,
	vmid => $vmid,
	node => $node,
	ssh_info => $ssh_info,
	nodeip => $nodeip,
	rem_ssh => PVE::Cluster::ssh_info_to_command($ssh_info)
    };

    $self = bless $self, $class;

    my $starttime = time();

    local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
	$self->log('err', "received interrupt - delayed");
	$self->{delayed_interrupt} = 1;
    };

    # lock container during migration
    eval { $self->lock_vm($self->{vmid}, sub {

	$self->{running} = 0;
	&$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
	die $@ if $@;

	if (defined($migration_network)) {
	    $self->log('info', "use dedicated network address for sending " .
	               "migration traffic ($self->{nodeip})");

	    # test if we can connect to new IP
	    my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
	    eval { $self->cmd_quiet($cmd); };
	    die "Can't connect to destination address ($self->{nodeip}) using " .
	        "public key authentication\n" if $@;
	}

	&$eval_int($self, sub { $self->phase1($self->{vmid}); });
	my $err = $@;
	if ($err) {
	    $self->log('err', $err);
	    eval { $self->phase1_cleanup($self->{vmid}, $err); };
	    if (my $tmperr = $@) {
		$self->log('err', $tmperr);
	    }
	    eval { $self->final_cleanup($self->{vmid}); };
	    if (my $tmperr = $@) {
		$self->log('err', $tmperr);
	    }
	    die $err;
	}

	# vm is now owned by other node
	# Note: there is no VM config file on the local node anymore

	if ($self->{running}) {

	    &$eval_int($self, sub { $self->phase2($self->{vmid}); });
	    my $phase2err = $@;
	    if ($phase2err) {
		$self->{errors} = 1;
		$self->log('err', "online migrate failure - $phase2err");
	    }
	    eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
	    if (my $err = $@) {
		$self->log('err', $err);
		$self->{errors} = 1;
	    }
	}

	# phase3 (finalize) 
	&$eval_int($self, sub { $self->phase3($self->{vmid}); });
	my $phase3err = $@;
	if ($phase3err) {
	    $self->log('err', $phase3err);
	    $self->{errors} = 1;
	}
	eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
	if (my $err = $@) {
	    $self->log('err', $err);
	    $self->{errors} = 1;
	}
	eval { $self->final_cleanup($self->{vmid}); };
	if (my $err = $@) {
	    $self->log('err', $err);
	    $self->{errors} = 1;
	}
    })};

    my $err = $@;

    my $delay = time() - $starttime;
    my $mins = int($delay/60);
    my $secs = $delay - $mins*60;
    my $hours =  int($mins/60);
    $mins = $mins - $hours*60;

    my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;

    if ($err) {
	$self->log('err', "migration aborted (duration $duration): $err");
	die "migration aborted\n";
    }

    if ($self->{errors}) {
	$self->log('err', "migration finished with problems (duration $duration)");
	die "migration problems\n"
    }

    $self->log('info', "migration finished successfully (duration $duration)");
}

sub lock_vm {
    my ($self, $vmid, $code, @param) = @_;

    die "abstract method - implement me";
}

sub prepare {
    my ($self, $vmid) = @_;

    die "abstract method - implement me";

    # return $running;
}

# transfer all data and move VM config files
sub phase1 {
    my ($self, $vmid) = @_;
    die "abstract method - implement me";
}

# only called if there are errors in phase1
sub phase1_cleanup {
    my ($self, $vmid, $err) = @_;
    die "abstract method - implement me";
}

# only called when VM is running and phase1 was successful
sub phase2 {
    my ($self, $vmid) = @_;
    die "abstract method - implement me";
}

# only called when VM is running and phase1 was successful
sub phase2_cleanup {
    my ($self, $vmid, $err) = @_;
};

#  only called when phase1 was successful
sub phase3 {
    my ($self, $vmid) = @_;
}

#  only called when phase1 was successful
sub phase3_cleanup {
    my ($self, $vmid, $err) = @_;
}

# final cleanup - always called
sub final_cleanup {
    my ($self, $vmid) = @_;
    die "abstract method - implement me";
}

# transfer replication state helper - call this before moving the guest config
# - just log the error is something fails
sub transfer_replication_state {
    my ($self) = @_;

    my $local_node = PVE::INotify::nodename();

    eval {
	my $stateobj = PVE::ReplicationState::read_state();
	my $vmstate = PVE::ReplicationState::extract_vmid_tranfer_state($stateobj, $self->{vmid}, $self->{node}, $local_node);
	# This have to be quoted when it run it over ssh.
	my $state = PVE::Tools::shellquote(encode_json($vmstate));
	my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $self->{vmid}, $state];
	$self->cmd($cmd);
    };
    if (my $err = $@) {
	$self->log('err', "transfer replication state failed - $err");
	$self->{errors} = 1;
    }
}

# switch replication job target - call this after moving the guest config
# - does nothing if there is no relevant replication job
# - just log the error is something fails
sub switch_replication_job_target {
    my ($self) = @_;

    my $local_node = PVE::INotify::nodename();

    eval { PVE::ReplicationConfig::switch_replication_job_target($self->{vmid}, $self->{node}, $local_node); };
    if (my $err = $@) {
	$self->log('err', "switch replication job target failed - $err");
	$self->{errors} = 1;
    }
}

1;
Commit	Line	Data
58a3c91c FG	1	package PVE::AbstractMigrate;
	2
	3	use strict;
	4	use warnings;
	5	use POSIX qw(strftime);
1c960710	6	use JSON;
58a3c91c	7	use PVE::Tools;
9591475a	8	use PVE::Cluster;
1c960710	9	use PVE::ReplicationState;
58a3c91c FG	10
	11	my $msg2text = sub {
	12	my ($level, $msg) = @_;
	13
	14	chomp $msg;
	15
	16	return '' if !$msg;
	17
	18	my $res = '';
	19
9ed0fe88	20	my $tstr = strftime("%F %H:%M:%S", localtime);
58a3c91c FG	21
	22	foreach my $line (split (/\n/, $msg)) {
	23	if ($level eq 'err') {
	24	$res .= "$tstr ERROR: $line\n";
	25	} else {
	26	$res .= "$tstr $line\n";
	27	}
	28	}
	29
	30	return $res;
	31	};
	32
	33	sub log {
	34	my ($self, $level, $msg) = @_;
	35
	36	chomp $msg;
	37
	38	return if !$msg;
	39
	40	print &$msg2text($level, $msg);
	41	}
	42
	43	sub cmd {
	44	my ($self, $cmd, %param) = @_;
	45
	46	my $logfunc = sub {
	47	my $line = shift;
	48	$self->log('info', $line);
	49	};
	50
	51	$self->log('info', "# " . PVE::Tools::cmd2string($cmd));
	52
	53	PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
	54	}
	55
	56	my $run_command_quiet_full = sub {
	57	my ($self, $cmd, $logerr, %param) = @_;
	58
	59	my $log = '';
	60	my $logfunc = sub {
	61	my $line = shift;
	62	$log .= &$msg2text('info', $line);;
	63	};
	64
	65	eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
	66	if (my $err = $@) {
	67	$self->log('info', "# " . PVE::Tools::cmd2string($cmd));
	68	print $log;
	69	if ($logerr) {
	70	$self->{errors} = 1;
	71	$self->log('err', $err);
	72	} else {
	73	die $err;
	74	}
	75	}
	76	};
	77
	78	sub cmd_quiet {
	79	my ($self, $cmd, %param) = @_;
	80	return &$run_command_quiet_full($self, $cmd, 0, %param);
	81	}
	82
	83	sub cmd_logerr {
	84	my ($self, $cmd, %param) = @_;
85	return &$run_command_quiet_full($self, $cmd, 1, %param);
86	}
87
58a3c91c FG	88	my $eval_int = sub {
	89	my ($self, $func, @param) = @_;
	90
	91	eval {
	92	local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
	93	$self->{delayed_interrupt} = 0;
	94	die "interrupted by signal\n";
	95	};
	96	local $SIG{PIPE} = sub {
	97	$self->{delayed_interrupt} = 0;
	98	die "interrupted by signal\n";
	99	};
	100
	101	my $di = $self->{delayed_interrupt};
	102	$self->{delayed_interrupt} = 0;
	103
	104	die "interrupted by signal\n" if $di;
	105
	106	&$func($self, @param);
	107	};
	108	};
	109
	110	my @ssh_opts = ('-o', 'BatchMode=yes');
	111	my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
58a3c91c	112
9591475a	113	# FIXME: nodeip is now unused
58a3c91c FG	114	sub migrate {
	115	my ($class, $node, $nodeip, $vmid, $opts) = @_;
	116
	117	$class = ref($class) \|\| $class;
	118
40318e97 WB	119	my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
40318e97 WB	120
9591475a WB	121	my $migration_network = $opts->{migration_network};
9591475a WB	122	if (!defined($migration_network)) {
9591475a WB	123	$migration_network = $dc_conf->{migration}->{network};
	124	}
	125	my $ssh_info = PVE::Cluster::get_ssh_info($node, $migration_network);
	126	$nodeip = $ssh_info->{ip};
	127
40318e97 WB	128	my $migration_type = 'secure';
	129	if (defined($opts->{migration_type})) {
	130	$migration_type = $opts->{migration_type};
	131	} elsif (defined($dc_conf->{migration}->{type})) {
	132	$migration_type = $dc_conf->{migration}->{type};
	133	}
	134	$opts->{migration_type} = $migration_type;
	135
58a3c91c FG	136	my $self = {
	137	delayed_interrupt => 0,
	138	opts => $opts,
	139	vmid => $vmid,
	140	node => $node,
9591475a	141	ssh_info => $ssh_info,
58a3c91c	142	nodeip => $nodeip,
9591475a	143	rem_ssh => PVE::Cluster::ssh_info_to_command($ssh_info)
58a3c91c FG	144	};
	145
	146	$self = bless $self, $class;
	147
	148	my $starttime = time();
	149
58a3c91c FG	150	local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
	151	$self->log('err', "received interrupt - delayed");
	152	$self->{delayed_interrupt} = 1;
	153	};
	154
58a3c91c FG	155	# lock container during migration
	156	eval { $self->lock_vm($self->{vmid}, sub {
	157
	158	$self->{running} = 0;
	159	&$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
	160	die $@ if $@;
	161
9591475a	162	if (defined($migration_network)) {
58a3c91c FG	163	$self->log('info', "use dedicated network address for sending " .
	164	"migration traffic ($self->{nodeip})");
	165
	166	# test if we can connect to new IP
	167	my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
	168	eval { $self->cmd_quiet($cmd); };
	169	die "Can't connect to destination address ($self->{nodeip}) using " .
	170	"public key authentication\n" if $@;
	171	}
	172
	173	&$eval_int($self, sub { $self->phase1($self->{vmid}); });
	174	my $err = $@;
	175	if ($err) {
	176	$self->log('err', $err);
	177	eval { $self->phase1_cleanup($self->{vmid}, $err); };
	178	if (my $tmperr = $@) {
	179	$self->log('err', $tmperr);
	180	}
	181	eval { $self->final_cleanup($self->{vmid}); };
	182	if (my $tmperr = $@) {
	183	$self->log('err', $tmperr);
	184	}
	185	die $err;
	186	}
	187
	188	# vm is now owned by other node
	189	# Note: there is no VM config file on the local node anymore
	190
	191	if ($self->{running}) {
	192
	193	&$eval_int($self, sub { $self->phase2($self->{vmid}); });
	194	my $phase2err = $@;
	195	if ($phase2err) {
	196	$self->{errors} = 1;
	197	$self->log('err', "online migrate failure - $phase2err");
	198	}
	199	eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
	200	if (my $err = $@) {
	201	$self->log('err', $err);
	202	$self->{errors} = 1;
	203	}
	204	}
	205
	206	# phase3 (finalize)
	207	&$eval_int($self, sub { $self->phase3($self->{vmid}); });
	208	my $phase3err = $@;
	209	if ($phase3err) {
	210	$self->log('err', $phase3err);
	211	$self->{errors} = 1;
	212	}
	213	eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
	214	if (my $err = $@) {
	215	$self->log('err', $err);
	216	$self->{errors} = 1;
	217	}
	218	eval { $self->final_cleanup($self->{vmid}); };
	219	if (my $err = $@) {
	220	$self->log('err', $err);
	221	$self->{errors} = 1;
	222	}
	223	})};
	224
	225	my $err = $@;
	226
227	my $delay = time() - $starttime;
228	my $mins = int($delay/60);
229	my $secs = $delay - $mins*60;
230	my $hours = int($mins/60);
231	$mins = $mins - $hours*60;
232
233	my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;
234
235	if ($err) {
236	$self->log('err', "migration aborted (duration $duration): $err");
237	die "migration aborted\n";
238	}
239
240	if ($self->{errors}) {
241	$self->log('err', "migration finished with problems (duration $duration)");
242	die "migration problems\n"
243	}
244
245	$self->log('info', "migration finished successfully (duration $duration)");
246	}
247
248	sub lock_vm {
249	my ($self, $vmid, $code, @param) = @_;
250
251	die "abstract method - implement me";
252	}
253
254	sub prepare {
255	my ($self, $vmid) = @_;
256
257	die "abstract method - implement me";
258
259	# return $running;
260	}
261
262	# transfer all data and move VM config files
263	sub phase1 {
264	my ($self, $vmid) = @_;
265	die "abstract method - implement me";
266	}
267
268	# only called if there are errors in phase1
269	sub phase1_cleanup {
270	my ($self, $vmid, $err) = @_;
271	die "abstract method - implement me";
272	}
273
274	# only called when VM is running and phase1 was successful
275	sub phase2 {
276	my ($self, $vmid) = @_;
277	die "abstract method - implement me";
278	}
279
280	# only called when VM is running and phase1 was successful
281	sub phase2_cleanup {
282	my ($self, $vmid, $err) = @_;
283	};
284
285	# only called when phase1 was successful
286	sub phase3 {
287	my ($self, $vmid) = @_;
288	}
289
290	# only called when phase1 was successful
291	sub phase3_cleanup {
292	my ($self, $vmid, $err) = @_;
293	}
294
295	# final cleanup - always called
296	sub final_cleanup {
297	my ($self, $vmid) = @_;
298	die "abstract method - implement me";
299	}
300
1c960710 DM	301	# transfer replication state helper - call this before moving the guest config
	302	# - just log the error is something fails
	303	sub transfer_replication_state {
	304	my ($self) = @_;
	305
	306	my $local_node = PVE::INotify::nodename();
	307
	308	eval {
	309	my $stateobj = PVE::ReplicationState::read_state();
	310	my $vmstate = PVE::ReplicationState::extract_vmid_tranfer_state($stateobj, $self->{vmid}, $self->{node}, $local_node);
	311	# This have to be quoted when it run it over ssh.
	312	my $state = PVE::Tools::shellquote(encode_json($vmstate));
	313	my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $self->{vmid}, $state];
	314	$self->cmd($cmd);
	315	};
	316	if (my $err = $@) {
	317	$self->log('err', "transfer replication state failed - $err");
	318	$self->{errors} = 1;
	319	}
	320	}
	321
	322	# switch replication job target - call this after moving the guest config
	323	# - does nothing if there is no relevant replication job
	324	# - just log the error is something fails
	325	sub switch_replication_job_target {
	326	my ($self) = @_;
	327
	328	my $local_node = PVE::INotify::nodename();
	329
	330	eval { PVE::ReplicationConfig::switch_replication_job_target($self->{vmid}, $self->{node}, $local_node); };
	331	if (my $err = $@) {
	332	$self->log('err', "switch replication job target failed - $err");
	333	$self->{errors} = 1;
	334	}
	335	}
	336
58a3c91c	337	1;