PVE/AbstractMigrate.pm

   1 package PVE::AbstractMigrate;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(strftime);
   6 use JSON;
   7 use PVE::Tools;
   8 use PVE::Cluster;
   9 use PVE::DataCenterConfig;
  10 use PVE::ReplicationState;
  11 use PVE::SSHInfo;
  12
  13 my $msg2text = sub {
  14     my ($level, $msg) = @_;
  15
  16     chomp $msg;
  17
  18     return '' if !$msg;
  19
  20     my $res = '';
  21
  22     my $tstr = strftime("%F %H:%M:%S", localtime);
  23
  24     foreach my $line (split (/\n/, $msg)) {
  25         if ($level eq 'err') {
  26             $res .= "$tstr ERROR: $line\n";
  27         } else {
  28             $res .= "$tstr $line\n";
  29         }
  30     }
  31
  32     return $res;
  33 };
  34
  35 sub log {
  36     my ($self, $level, $msg) = @_;
  37
  38     chomp $msg;
  39
  40     return if !$msg;
  41
  42     print &$msg2text($level, $msg);
  43 }
  44
  45 sub cmd {
  46     my ($self, $cmd, %param) = @_;
  47
  48     my $logfunc = sub {
  49         my $line = shift;
  50         $self->log('info', $line);
  51     };
  52
  53     $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  54
  55     PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
  56 }
  57
  58 my $run_command_quiet_full = sub {
  59     my ($self, $cmd, $logerr, %param) = @_;
  60
  61     my $log = '';
  62     my $logfunc = sub {
  63         my $line = shift;
  64         $log .= &$msg2text('info', $line);;
  65     };
  66
  67     eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
  68     if (my $err = $@) {
  69         $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  70         print $log;
  71         if ($logerr) {
  72             $self->{errors} = 1;
  73             $self->log('err', $err);
  74         } else {
  75             die $err;
  76         }
  77     }
  78 };
  79
  80 sub cmd_quiet {
  81     my ($self, $cmd, %param) = @_;
  82     return &$run_command_quiet_full($self, $cmd, 0, %param);
  83 }
  84
  85 sub cmd_logerr {
  86     my ($self, $cmd, %param) = @_;
  87     return &$run_command_quiet_full($self, $cmd, 1, %param);
  88 }
  89
  90 my $eval_int = sub {
  91     my ($self, $func, @param) = @_;
  92
  93     eval {
  94         local $SIG{INT} =
  95             local $SIG{TERM} =
  96             local $SIG{QUIT} =
  97             local $SIG{HUP} =
  98             local $SIG{PIPE} = sub {
  99                 $self->{delayed_interrupt} = 0;
 100                 die "interrupted by signal\n";
 101         };
 102
 103         my $di = $self->{delayed_interrupt};
 104         $self->{delayed_interrupt} = 0;
 105
 106         die "interrupted by signal\n" if $di;
 107
 108         &$func($self, @param);
 109     };
 110 };
 111
 112 # FIXME: nodeip is now unused
 113 sub migrate {
 114     my ($class, $node, $nodeip, $vmid, $opts) = @_;
 115
 116     $class = ref($class) || $class;
 117
 118     my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 119
 120     my $migration_network = $opts->{migration_network};
 121     if (!defined($migration_network)) {
 122         $migration_network = $dc_conf->{migration}->{network};
 123     }
 124     my $ssh_info = PVE::SSHInfo::get_ssh_info($node, $migration_network);
 125     $nodeip = $ssh_info->{ip};
 126
 127     my $migration_type = 'secure';
 128     if (defined($opts->{migration_type})) {
 129         $migration_type = $opts->{migration_type};
 130     } elsif (defined($dc_conf->{migration}->{type})) {
 131         $migration_type = $dc_conf->{migration}->{type};
 132     }
 133     $opts->{migration_type} = $migration_type;
 134
 135     my $self = {
 136         delayed_interrupt => 0,
 137         opts => $opts,
 138         vmid => $vmid,
 139         node => $node,
 140         ssh_info => $ssh_info,
 141         nodeip => $nodeip,
 142         rem_ssh => PVE::SSHInfo::ssh_info_to_command($ssh_info)
 143     };
 144
 145     $self = bless $self, $class;
 146
 147     my $starttime = time();
 148
 149     local $SIG{INT} =
 150         local $SIG{TERM} =
 151         local $SIG{QUIT} =
 152         local $SIG{HUP} =
 153         local $SIG{PIPE} = sub {
 154             $self->log('err', "received interrupt - delayed");
 155             $self->{delayed_interrupt} = 1;
 156     };
 157
 158     # lock container during migration
 159     eval { $self->lock_vm($self->{vmid}, sub {
 160
 161         $self->{running} = 0;
 162         &$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
 163         die $@ if $@;
 164
 165         if (defined($migration_network)) {
 166             $self->log('info', "use dedicated network address for sending " .
 167                        "migration traffic ($self->{nodeip})");
 168
 169             # test if we can connect to new IP
 170             my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
 171             eval { $self->cmd_quiet($cmd); };
 172             die "Can't connect to destination address ($self->{nodeip}) using " .
 173                 "public key authentication\n" if $@;
 174         }
 175
 176         &$eval_int($self, sub { $self->phase1($self->{vmid}); });
 177         my $err = $@;
 178         if ($err) {
 179             $self->log('err', $err);
 180             eval { $self->phase1_cleanup($self->{vmid}, $err); };
 181             if (my $tmperr = $@) {
 182                 $self->log('err', $tmperr);
 183             }
 184             eval { $self->final_cleanup($self->{vmid}); };
 185             if (my $tmperr = $@) {
 186                 $self->log('err', $tmperr);
 187             }
 188             die $err;
 189         }
 190
 191         # vm is now owned by other node
 192         # Note: there is no VM config file on the local node anymore
 193
 194         if ($self->{running}) {
 195
 196             &$eval_int($self, sub { $self->phase2($self->{vmid}); });
 197             my $phase2err = $@;
 198             if ($phase2err) {
 199                 $self->{errors} = 1;
 200                 $self->log('err', "online migrate failure - $phase2err");
 201             }
 202             eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
 203             if (my $err = $@) {
 204                 $self->log('err', $err);
 205                 $self->{errors} = 1;
 206             }
 207         }
 208
 209         # phase3 (finalize)
 210         &$eval_int($self, sub { $self->phase3($self->{vmid}); });
 211         my $phase3err = $@;
 212         if ($phase3err) {
 213             $self->log('err', $phase3err);
 214             $self->{errors} = 1;
 215         }
 216         eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
 217         if (my $err = $@) {
 218             $self->log('err', $err);
 219             $self->{errors} = 1;
 220         }
 221         eval { $self->final_cleanup($self->{vmid}); };
 222         if (my $err = $@) {
 223             $self->log('err', $err);
 224             $self->{errors} = 1;
 225         }
 226     })};
 227
 228     my $err = $@;
 229
 230     my $delay = time() - $starttime;
 231     my $mins = int($delay/60);
 232     my $secs = $delay - $mins*60;
 233     my $hours =  int($mins/60);
 234     $mins = $mins - $hours*60;
 235
 236     my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;
 237
 238     if ($err) {
 239         $self->log('err', "migration aborted (duration $duration): $err");
 240         die "migration aborted\n";
 241     }
 242
 243     if ($self->{errors}) {
 244         $self->log('err', "migration finished with problems (duration $duration)");
 245         die "migration problems\n"
 246     }
 247
 248     $self->log('info', "migration finished successfully (duration $duration)");
 249 }
 250
 251 sub lock_vm {
 252     my ($self, $vmid, $code, @param) = @_;
 253
 254     die "abstract method - implement me";
 255 }
 256
 257 sub prepare {
 258     my ($self, $vmid) = @_;
 259
 260     die "abstract method - implement me";
 261
 262     # return $running;
 263 }
 264
 265 # transfer all data and move VM config files
 266 sub phase1 {
 267     my ($self, $vmid) = @_;
 268     die "abstract method - implement me";
 269 }
 270
 271 # only called if there are errors in phase1
 272 sub phase1_cleanup {
 273     my ($self, $vmid, $err) = @_;
 274     die "abstract method - implement me";
 275 }
 276
 277 # only called when VM is running and phase1 was successful
 278 sub phase2 {
 279     my ($self, $vmid) = @_;
 280     die "abstract method - implement me";
 281 }
 282
 283 # only called when VM is running and phase1 was successful
 284 sub phase2_cleanup {
 285     my ($self, $vmid, $err) = @_;
 286 };
 287
 288 #  only called when phase1 was successful
 289 sub phase3 {
 290     my ($self, $vmid) = @_;
 291 }
 292
 293 #  only called when phase1 was successful
 294 sub phase3_cleanup {
 295     my ($self, $vmid, $err) = @_;
 296 }
 297
 298 # final cleanup - always called
 299 sub final_cleanup {
 300     my ($self, $vmid) = @_;
 301     die "abstract method - implement me";
 302 }
 303
 304 # transfer replication state helper - call this before moving the guest config
 305 # - just log the error is something fails
 306 sub transfer_replication_state {
 307     my ($self) = @_;
 308
 309     my $local_node = PVE::INotify::nodename();
 310
 311     eval {
 312         my $stateobj = PVE::ReplicationState::read_state();
 313         my $vmstate = PVE::ReplicationState::extract_vmid_tranfer_state($stateobj, $self->{vmid}, $self->{node}, $local_node);
 314         # This have to be quoted when it run it over ssh.
 315         my $state = PVE::Tools::shellquote(encode_json($vmstate));
 316         my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $self->{vmid}, $state];
 317         $self->cmd($cmd);
 318     };
 319     if (my $err = $@) {
 320         $self->log('err', "transfer replication state failed - $err");
 321         $self->{errors} = 1;
 322     }
 323 }
 324
 325 # switch replication job target - call this after moving the guest config
 326 # - does nothing if there is no relevant replication job
 327 # - just log the error is something fails
 328 sub switch_replication_job_target {
 329     my ($self) = @_;
 330
 331     my $local_node = PVE::INotify::nodename();
 332
 333     eval { PVE::ReplicationConfig::switch_replication_job_target($self->{vmid}, $self->{node}, $local_node); };
 334     if (my $err = $@) {
 335         $self->log('err', "switch replication job target failed - $err");
 336         $self->{errors} = 1;
 337     }
 338 }
 339
 340 1;