src/PVE/AbstractMigrate.pm

   1 package PVE::AbstractMigrate;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(strftime);
   6 use PVE::Tools;
   7
   8 my $msg2text = sub {
   9     my ($level, $msg) = @_;
  10
  11     chomp $msg;
  12
  13     return '' if !$msg;
  14
  15     my $res = '';
  16
  17     my $tstr = strftime("%b %d %H:%M:%S", localtime);
  18
  19     foreach my $line (split (/\n/, $msg)) {
  20         if ($level eq 'err') {
  21             $res .= "$tstr ERROR: $line\n";
  22         } else {
  23             $res .= "$tstr $line\n";
  24         }
  25     }
  26
  27     return $res;
  28 };
  29
  30 sub log {
  31     my ($self, $level, $msg) = @_;
  32
  33     chomp $msg;
  34
  35     return if !$msg;
  36
  37     print &$msg2text($level, $msg);
  38 }
  39
  40 sub cmd {
  41     my ($self, $cmd, %param) = @_;
  42
  43     my $logfunc = sub {
  44         my $line = shift;
  45         $self->log('info', $line);
  46     };
  47
  48     $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  49
  50     PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
  51 }
  52
  53 my $run_command_quiet_full = sub {
  54     my ($self, $cmd, $logerr, %param) = @_;
  55
  56     my $log = '';
  57     my $logfunc = sub {
  58         my $line = shift;
  59         $log .= &$msg2text('info', $line);;
  60     };
  61
  62     eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
  63     if (my $err = $@) {
  64         $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  65         print $log;
  66         if ($logerr) {
  67             $self->{errors} = 1;
  68             $self->log('err', $err);
  69         } else {
  70             die $err;
  71         }
  72     }
  73 };
  74
  75 sub cmd_quiet {
  76     my ($self, $cmd, %param) = @_;
  77     return &$run_command_quiet_full($self, $cmd, 0, %param);
  78 }
  79
  80 sub cmd_logerr {
  81     my ($self, $cmd, %param) = @_;
  82     return &$run_command_quiet_full($self, $cmd, 1, %param);
  83 }
  84
  85 sub get_remote_migration_ip {
  86     my ($self) = @_;
  87
  88     my $ip;
  89
  90     my $cmd = [@{$self->{rem_ssh}}, 'pvecm', 'mtunnel', '--get_migration_ip'];
  91
  92     push @$cmd, '--migration_network', $self->{opts}->{migration_network}
  93       if defined($self->{opts}->{migration_network});
  94
  95     PVE::Tools::run_command($cmd, outfunc => sub {
  96         my $line = shift;
  97
  98         if ($line =~ m/^ip: '($PVE::Tools::IPRE)'$/) {
  99            $ip = $1;
 100         }
 101     });
 102
 103     return $ip;
 104 }
 105
 106 my $eval_int = sub {
 107     my ($self, $func, @param) = @_;
 108
 109     eval {
 110         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
 111             $self->{delayed_interrupt} = 0;
 112             die "interrupted by signal\n";
 113         };
 114         local $SIG{PIPE} = sub {
 115             $self->{delayed_interrupt} = 0;
 116             die "interrupted by signal\n";
 117         };
 118
 119         my $di = $self->{delayed_interrupt};
 120         $self->{delayed_interrupt} = 0;
 121
 122         die "interrupted by signal\n" if $di;
 123
 124         &$func($self, @param);
 125     };
 126 };
 127
 128 my @ssh_opts = ('-o', 'BatchMode=yes');
 129 my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
 130 my @scp_cmd = ('/usr/bin/scp', @ssh_opts);
 131 my @rsync_opts = ('-aHAX', '--delete', '--numeric-ids');
 132 my @rsync_cmd = ('/usr/bin/rsync', @rsync_opts);
 133
 134 sub migrate {
 135     my ($class, $node, $nodeip, $vmid, $opts) = @_;
 136
 137     $class = ref($class) || $class;
 138
 139     my $self = {
 140         delayed_interrupt => 0,
 141         opts => $opts,
 142         vmid => $vmid,
 143         node => $node,
 144         nodeip => $nodeip,
 145         rsync_cmd => [ @rsync_cmd ],
 146         rem_ssh => [ @ssh_cmd, "root\@$nodeip" ],
 147         scp_cmd => [ @scp_cmd ],
 148     };
 149
 150     $self = bless $self, $class;
 151
 152     my $starttime = time();
 153
 154     local $ENV{RSYNC_RSH} = join(' ', @ssh_cmd);
 155
 156     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 157         $self->log('err', "received interrupt - delayed");
 158         $self->{delayed_interrupt} = 1;
 159     };
 160
 161     local $ENV{RSYNC_RSH} = join(' ', @ssh_cmd);
 162
 163     # lock container during migration
 164     eval { $self->lock_vm($self->{vmid}, sub {
 165
 166         $self->{running} = 0;
 167         &$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
 168         die $@ if $@;
 169
 170         # get dedicated migration address from remote node, if set.
 171         # as a side effect this checks also if the other node can be accessed
 172         # through ssh and that it has quorum
 173         my $remote_migration_ip = $self->get_remote_migration_ip();
 174
 175         if (defined($remote_migration_ip)) {
 176             $nodeip = $remote_migration_ip;
 177             $self->{nodeip} = $remote_migration_ip;
 178             $self->{rem_ssh} = [ @ssh_cmd, "root\@$nodeip" ];
 179
 180             $self->log('info', "use dedicated network address for sending " .
 181                        "migration traffic ($self->{nodeip})");
 182
 183             # test if we can connect to new IP
 184             my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
 185             eval { $self->cmd_quiet($cmd); };
 186             die "Can't connect to destination address ($self->{nodeip}) using " .
 187                 "public key authentication\n" if $@;
 188         }
 189
 190         &$eval_int($self, sub { $self->phase1($self->{vmid}); });
 191         my $err = $@;
 192         if ($err) {
 193             $self->log('err', $err);
 194             eval { $self->phase1_cleanup($self->{vmid}, $err); };
 195             if (my $tmperr = $@) {
 196                 $self->log('err', $tmperr);
 197             }
 198             eval { $self->final_cleanup($self->{vmid}); };
 199             if (my $tmperr = $@) {
 200                 $self->log('err', $tmperr);
 201             }
 202             die $err;
 203         }
 204
 205         # vm is now owned by other node
 206         # Note: there is no VM config file on the local node anymore
 207
 208         if ($self->{running}) {
 209
 210             &$eval_int($self, sub { $self->phase2($self->{vmid}); });
 211             my $phase2err = $@;
 212             if ($phase2err) {
 213                 $self->{errors} = 1;
 214                 $self->log('err', "online migrate failure - $phase2err");
 215             }
 216             eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
 217             if (my $err = $@) {
 218                 $self->log('err', $err);
 219                 $self->{errors} = 1;
 220             }
 221         }
 222
 223         # phase3 (finalize)
 224         &$eval_int($self, sub { $self->phase3($self->{vmid}); });
 225         my $phase3err = $@;
 226         if ($phase3err) {
 227             $self->log('err', $phase3err);
 228             $self->{errors} = 1;
 229         }
 230         eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
 231         if (my $err = $@) {
 232             $self->log('err', $err);
 233             $self->{errors} = 1;
 234         }
 235         eval { $self->final_cleanup($self->{vmid}); };
 236         if (my $err = $@) {
 237             $self->log('err', $err);
 238             $self->{errors} = 1;
 239         }
 240     })};
 241
 242     my $err = $@;
 243
 244     my $delay = time() - $starttime;
 245     my $mins = int($delay/60);
 246     my $secs = $delay - $mins*60;
 247     my $hours =  int($mins/60);
 248     $mins = $mins - $hours*60;
 249
 250     my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;
 251
 252     if ($err) {
 253         $self->log('err', "migration aborted (duration $duration): $err");
 254         die "migration aborted\n";
 255     }
 256
 257     if ($self->{errors}) {
 258         $self->log('err', "migration finished with problems (duration $duration)");
 259         die "migration problems\n"
 260     }
 261
 262     $self->log('info', "migration finished successfully (duration $duration)");
 263 }
 264
 265 sub lock_vm {
 266     my ($self, $vmid, $code, @param) = @_;
 267
 268     die "abstract method - implement me";
 269 }
 270
 271 sub prepare {
 272     my ($self, $vmid) = @_;
 273
 274     die "abstract method - implement me";
 275
 276     # return $running;
 277 }
 278
 279 # transfer all data and move VM config files
 280 sub phase1 {
 281     my ($self, $vmid) = @_;
 282     die "abstract method - implement me";
 283 }
 284
 285 # only called if there are errors in phase1
 286 sub phase1_cleanup {
 287     my ($self, $vmid, $err) = @_;
 288     die "abstract method - implement me";
 289 }
 290
 291 # only called when VM is running and phase1 was successful
 292 sub phase2 {
 293     my ($self, $vmid) = @_;
 294     die "abstract method - implement me";
 295 }
 296
 297 # only called when VM is running and phase1 was successful
 298 sub phase2_cleanup {
 299     my ($self, $vmid, $err) = @_;
 300 };
 301
 302 #  only called when phase1 was successful
 303 sub phase3 {
 304     my ($self, $vmid) = @_;
 305 }
 306
 307 #  only called when phase1 was successful
 308 sub phase3_cleanup {
 309     my ($self, $vmid, $err) = @_;
 310 }
 311
 312 # final cleanup - always called
 313 sub final_cleanup {
 314     my ($self, $vmid) = @_;
 315     die "abstract method - implement me";
 316 }
 317
 318 1;