src/PVE/AbstractMigrate.pm

   1 package PVE::AbstractMigrate;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(strftime);
   6 use PVE::Tools;
   7
   8 my $msg2text = sub {
   9     my ($level, $msg) = @_;
  10
  11     chomp $msg;
  12
  13     return '' if !$msg;
  14
  15     my $res = '';
  16
  17     my $tstr = strftime("%b %d %H:%M:%S", localtime);
  18
  19     foreach my $line (split (/\n/, $msg)) {
  20         if ($level eq 'err') {
  21             $res .= "$tstr ERROR: $line\n";
  22         } else {
  23             $res .= "$tstr $line\n";
  24         }
  25     }
  26
  27     return $res;
  28 };
  29
  30 sub log {
  31     my ($self, $level, $msg) = @_;
  32
  33     chomp $msg;
  34
  35     return if !$msg;
  36
  37     print &$msg2text($level, $msg);
  38 }
  39
  40 sub cmd {
  41     my ($self, $cmd, %param) = @_;
  42
  43     my $logfunc = sub {
  44         my $line = shift;
  45         $self->log('info', $line);
  46     };
  47
  48     $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  49
  50     PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
  51 }
  52
  53 my $run_command_quiet_full = sub {
  54     my ($self, $cmd, $logerr, %param) = @_;
  55
  56     my $log = '';
  57     my $logfunc = sub {
  58         my $line = shift;
  59         $log .= &$msg2text('info', $line);;
  60     };
  61
  62     eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
  63     if (my $err = $@) {
  64         $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  65         print $log;
  66         if ($logerr) {
  67             $self->{errors} = 1;
  68             $self->log('err', $err);
  69         } else {
  70             die $err;
  71         }
  72     }
  73 };
  74
  75 sub cmd_quiet {
  76     my ($self, $cmd, %param) = @_;
  77     return &$run_command_quiet_full($self, $cmd, 0, %param);
  78 }
  79
  80 sub cmd_logerr {
  81     my ($self, $cmd, %param) = @_;
  82     return &$run_command_quiet_full($self, $cmd, 1, %param);
  83 }
  84
  85 sub get_remote_migration_ip {
  86     my ($self) = @_;
  87
  88     my $ip;
  89
  90     my $cmd = [@{$self->{rem_ssh}}, 'pvecm', 'mtunnel', '--get_migration_ip'];
  91
  92     push @$cmd, '--migration_network', $self->{opts}->{migration_network}
  93       if defined($self->{opts}->{migration_network});
  94
  95     PVE::Tools::run_command($cmd, outfunc => sub {
  96         my $line = shift;
  97
  98         # use non-restrictive regex for ip, its already checked by the remote side
  99         if ($line =~ m/^ip: '(\S+)'$/) {
 100            $ip = $1;
 101         }
 102     });
 103
 104     return $ip;
 105 }
 106
 107 my $eval_int = sub {
 108     my ($self, $func, @param) = @_;
 109
 110     eval {
 111         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
 112             $self->{delayed_interrupt} = 0;
 113             die "interrupted by signal\n";
 114         };
 115         local $SIG{PIPE} = sub {
 116             $self->{delayed_interrupt} = 0;
 117             die "interrupted by signal\n";
 118         };
 119
 120         my $di = $self->{delayed_interrupt};
 121         $self->{delayed_interrupt} = 0;
 122
 123         die "interrupted by signal\n" if $di;
 124
 125         &$func($self, @param);
 126     };
 127 };
 128
 129 my @ssh_opts = ('-o', 'BatchMode=yes');
 130 my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
 131 my @scp_cmd = ('/usr/bin/scp', @ssh_opts);
 132 my @rsync_opts = ('-aHAX', '--delete', '--numeric-ids');
 133 my @rsync_cmd = ('/usr/bin/rsync', @rsync_opts);
 134
 135 sub migrate {
 136     my ($class, $node, $nodeip, $vmid, $opts) = @_;
 137
 138     $class = ref($class) || $class;
 139
 140     my $self = {
 141         delayed_interrupt => 0,
 142         opts => $opts,
 143         vmid => $vmid,
 144         node => $node,
 145         nodeip => $nodeip,
 146         rsync_cmd => [ @rsync_cmd ],
 147         rem_ssh => [ @ssh_cmd, "root\@$nodeip" ],
 148         scp_cmd => [ @scp_cmd ],
 149     };
 150
 151     $self = bless $self, $class;
 152
 153     my $starttime = time();
 154
 155     local $ENV{RSYNC_RSH} = join(' ', @ssh_cmd);
 156
 157     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 158         $self->log('err', "received interrupt - delayed");
 159         $self->{delayed_interrupt} = 1;
 160     };
 161
 162     local $ENV{RSYNC_RSH} = join(' ', @ssh_cmd);
 163
 164     # lock container during migration
 165     eval { $self->lock_vm($self->{vmid}, sub {
 166
 167         $self->{running} = 0;
 168         &$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
 169         die $@ if $@;
 170
 171         # get dedicated migration address from remote node, if set.
 172         # as a side effect this checks also if the other node can be accessed
 173         # through ssh and that it has quorum
 174         my $remote_migration_ip = $self->get_remote_migration_ip();
 175
 176         if (defined($remote_migration_ip)) {
 177             $nodeip = $remote_migration_ip;
 178             $self->{nodeip} = $remote_migration_ip;
 179             $self->{rem_ssh} = [ @ssh_cmd, "root\@$nodeip" ];
 180
 181             $self->log('info', "use dedicated network address for sending " .
 182                        "migration traffic ($self->{nodeip})");
 183
 184             # test if we can connect to new IP
 185             my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
 186             eval { $self->cmd_quiet($cmd); };
 187             die "Can't connect to destination address ($self->{nodeip}) using " .
 188                 "public key authentication\n" if $@;
 189         }
 190
 191         &$eval_int($self, sub { $self->phase1($self->{vmid}); });
 192         my $err = $@;
 193         if ($err) {
 194             $self->log('err', $err);
 195             eval { $self->phase1_cleanup($self->{vmid}, $err); };
 196             if (my $tmperr = $@) {
 197                 $self->log('err', $tmperr);
 198             }
 199             eval { $self->final_cleanup($self->{vmid}); };
 200             if (my $tmperr = $@) {
 201                 $self->log('err', $tmperr);
 202             }
 203             die $err;
 204         }
 205
 206         # vm is now owned by other node
 207         # Note: there is no VM config file on the local node anymore
 208
 209         if ($self->{running}) {
 210
 211             &$eval_int($self, sub { $self->phase2($self->{vmid}); });
 212             my $phase2err = $@;
 213             if ($phase2err) {
 214                 $self->{errors} = 1;
 215                 $self->log('err', "online migrate failure - $phase2err");
 216             }
 217             eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
 218             if (my $err = $@) {
 219                 $self->log('err', $err);
 220                 $self->{errors} = 1;
 221             }
 222         }
 223
 224         # phase3 (finalize)
 225         &$eval_int($self, sub { $self->phase3($self->{vmid}); });
 226         my $phase3err = $@;
 227         if ($phase3err) {
 228             $self->log('err', $phase3err);
 229             $self->{errors} = 1;
 230         }
 231         eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
 232         if (my $err = $@) {
 233             $self->log('err', $err);
 234             $self->{errors} = 1;
 235         }
 236         eval { $self->final_cleanup($self->{vmid}); };
 237         if (my $err = $@) {
 238             $self->log('err', $err);
 239             $self->{errors} = 1;
 240         }
 241     })};
 242
 243     my $err = $@;
 244
 245     my $delay = time() - $starttime;
 246     my $mins = int($delay/60);
 247     my $secs = $delay - $mins*60;
 248     my $hours =  int($mins/60);
 249     $mins = $mins - $hours*60;
 250
 251     my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;
 252
 253     if ($err) {
 254         $self->log('err', "migration aborted (duration $duration): $err");
 255         die "migration aborted\n";
 256     }
 257
 258     if ($self->{errors}) {
 259         $self->log('err', "migration finished with problems (duration $duration)");
 260         die "migration problems\n"
 261     }
 262
 263     $self->log('info', "migration finished successfully (duration $duration)");
 264 }
 265
 266 sub lock_vm {
 267     my ($self, $vmid, $code, @param) = @_;
 268
 269     die "abstract method - implement me";
 270 }
 271
 272 sub prepare {
 273     my ($self, $vmid) = @_;
 274
 275     die "abstract method - implement me";
 276
 277     # return $running;
 278 }
 279
 280 # transfer all data and move VM config files
 281 sub phase1 {
 282     my ($self, $vmid) = @_;
 283     die "abstract method - implement me";
 284 }
 285
 286 # only called if there are errors in phase1
 287 sub phase1_cleanup {
 288     my ($self, $vmid, $err) = @_;
 289     die "abstract method - implement me";
 290 }
 291
 292 # only called when VM is running and phase1 was successful
 293 sub phase2 {
 294     my ($self, $vmid) = @_;
 295     die "abstract method - implement me";
 296 }
 297
 298 # only called when VM is running and phase1 was successful
 299 sub phase2_cleanup {
 300     my ($self, $vmid, $err) = @_;
 301 };
 302
 303 #  only called when phase1 was successful
 304 sub phase3 {
 305     my ($self, $vmid) = @_;
 306 }
 307
 308 #  only called when phase1 was successful
 309 sub phase3_cleanup {
 310     my ($self, $vmid, $err) = @_;
 311 }
 312
 313 # final cleanup - always called
 314 sub final_cleanup {
 315     my ($self, $vmid) = @_;
 316     die "abstract method - implement me";
 317 }
 318
 319 1;