PVE/AbstractMigrate.pm

   1 package PVE::AbstractMigrate;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(strftime);
   6 use JSON;
   7 use PVE::Tools;
   8 use PVE::Cluster;
   9 use PVE::ReplicationState;
  10
  11 my $msg2text = sub {
  12     my ($level, $msg) = @_;
  13
  14     chomp $msg;
  15
  16     return '' if !$msg;
  17
  18     my $res = '';
  19
  20     my $tstr = strftime("%F %H:%M:%S", localtime);
  21
  22     foreach my $line (split (/\n/, $msg)) {
  23         if ($level eq 'err') {
  24             $res .= "$tstr ERROR: $line\n";
  25         } else {
  26             $res .= "$tstr $line\n";
  27         }
  28     }
  29
  30     return $res;
  31 };
  32
  33 sub log {
  34     my ($self, $level, $msg) = @_;
  35
  36     chomp $msg;
  37
  38     return if !$msg;
  39
  40     print &$msg2text($level, $msg);
  41 }
  42
  43 sub cmd {
  44     my ($self, $cmd, %param) = @_;
  45
  46     my $logfunc = sub {
  47         my $line = shift;
  48         $self->log('info', $line);
  49     };
  50
  51     $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  52
  53     PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
  54 }
  55
  56 my $run_command_quiet_full = sub {
  57     my ($self, $cmd, $logerr, %param) = @_;
  58
  59     my $log = '';
  60     my $logfunc = sub {
  61         my $line = shift;
  62         $log .= &$msg2text('info', $line);;
  63     };
  64
  65     eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
  66     if (my $err = $@) {
  67         $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  68         print $log;
  69         if ($logerr) {
  70             $self->{errors} = 1;
  71             $self->log('err', $err);
  72         } else {
  73             die $err;
  74         }
  75     }
  76 };
  77
  78 sub cmd_quiet {
  79     my ($self, $cmd, %param) = @_;
  80     return &$run_command_quiet_full($self, $cmd, 0, %param);
  81 }
  82
  83 sub cmd_logerr {
  84     my ($self, $cmd, %param) = @_;
  85     return &$run_command_quiet_full($self, $cmd, 1, %param);
  86 }
  87
  88 my $eval_int = sub {
  89     my ($self, $func, @param) = @_;
  90
  91     eval {
  92         local $SIG{INT} =
  93             local $SIG{TERM} =
  94             local $SIG{QUIT} =
  95             local $SIG{HUP} =
  96             local $SIG{PIPE} = sub {
  97                 $self->{delayed_interrupt} = 0;
  98                 die "interrupted by signal\n";
  99         };
 100
 101         my $di = $self->{delayed_interrupt};
 102         $self->{delayed_interrupt} = 0;
 103
 104         die "interrupted by signal\n" if $di;
 105
 106         &$func($self, @param);
 107     };
 108 };
 109
 110 # FIXME: nodeip is now unused
 111 sub migrate {
 112     my ($class, $node, $nodeip, $vmid, $opts) = @_;
 113
 114     $class = ref($class) || $class;
 115
 116     my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 117
 118     my $migration_network = $opts->{migration_network};
 119     if (!defined($migration_network)) {
 120         $migration_network = $dc_conf->{migration}->{network};
 121     }
 122     my $ssh_info = PVE::Cluster::get_ssh_info($node, $migration_network);
 123     $nodeip = $ssh_info->{ip};
 124
 125     my $migration_type = 'secure';
 126     if (defined($opts->{migration_type})) {
 127         $migration_type = $opts->{migration_type};
 128     } elsif (defined($dc_conf->{migration}->{type})) {
 129         $migration_type = $dc_conf->{migration}->{type};
 130     }
 131     $opts->{migration_type} = $migration_type;
 132
 133     my $self = {
 134         delayed_interrupt => 0,
 135         opts => $opts,
 136         vmid => $vmid,
 137         node => $node,
 138         ssh_info => $ssh_info,
 139         nodeip => $nodeip,
 140         rem_ssh => PVE::Cluster::ssh_info_to_command($ssh_info)
 141     };
 142
 143     $self = bless $self, $class;
 144
 145     my $starttime = time();
 146
 147     local $SIG{INT} =
 148         local $SIG{TERM} =
 149         local $SIG{QUIT} =
 150         local $SIG{HUP} =
 151         local $SIG{PIPE} = sub {
 152             $self->log('err', "received interrupt - delayed");
 153             $self->{delayed_interrupt} = 1;
 154     };
 155
 156     # lock container during migration
 157     eval { $self->lock_vm($self->{vmid}, sub {
 158
 159         $self->{running} = 0;
 160         &$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
 161         die $@ if $@;
 162
 163         if (defined($migration_network)) {
 164             $self->log('info', "use dedicated network address for sending " .
 165                        "migration traffic ($self->{nodeip})");
 166
 167             # test if we can connect to new IP
 168             my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
 169             eval { $self->cmd_quiet($cmd); };
 170             die "Can't connect to destination address ($self->{nodeip}) using " .
 171                 "public key authentication\n" if $@;
 172         }
 173
 174         &$eval_int($self, sub { $self->phase1($self->{vmid}); });
 175         my $err = $@;
 176         if ($err) {
 177             $self->log('err', $err);
 178             eval { $self->phase1_cleanup($self->{vmid}, $err); };
 179             if (my $tmperr = $@) {
 180                 $self->log('err', $tmperr);
 181             }
 182             eval { $self->final_cleanup($self->{vmid}); };
 183             if (my $tmperr = $@) {
 184                 $self->log('err', $tmperr);
 185             }
 186             die $err;
 187         }
 188
 189         # vm is now owned by other node
 190         # Note: there is no VM config file on the local node anymore
 191
 192         if ($self->{running}) {
 193
 194             &$eval_int($self, sub { $self->phase2($self->{vmid}); });
 195             my $phase2err = $@;
 196             if ($phase2err) {
 197                 $self->{errors} = 1;
 198                 $self->log('err', "online migrate failure - $phase2err");
 199             }
 200             eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
 201             if (my $err = $@) {
 202                 $self->log('err', $err);
 203                 $self->{errors} = 1;
 204             }
 205         }
 206
 207         # phase3 (finalize)
 208         &$eval_int($self, sub { $self->phase3($self->{vmid}); });
 209         my $phase3err = $@;
 210         if ($phase3err) {
 211             $self->log('err', $phase3err);
 212             $self->{errors} = 1;
 213         }
 214         eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
 215         if (my $err = $@) {
 216             $self->log('err', $err);
 217             $self->{errors} = 1;
 218         }
 219         eval { $self->final_cleanup($self->{vmid}); };
 220         if (my $err = $@) {
 221             $self->log('err', $err);
 222             $self->{errors} = 1;
 223         }
 224     })};
 225
 226     my $err = $@;
 227
 228     my $delay = time() - $starttime;
 229     my $mins = int($delay/60);
 230     my $secs = $delay - $mins*60;
 231     my $hours =  int($mins/60);
 232     $mins = $mins - $hours*60;
 233
 234     my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;
 235
 236     if ($err) {
 237         $self->log('err', "migration aborted (duration $duration): $err");
 238         die "migration aborted\n";
 239     }
 240
 241     if ($self->{errors}) {
 242         $self->log('err', "migration finished with problems (duration $duration)");
 243         die "migration problems\n"
 244     }
 245
 246     $self->log('info', "migration finished successfully (duration $duration)");
 247 }
 248
 249 sub lock_vm {
 250     my ($self, $vmid, $code, @param) = @_;
 251
 252     die "abstract method - implement me";
 253 }
 254
 255 sub prepare {
 256     my ($self, $vmid) = @_;
 257
 258     die "abstract method - implement me";
 259
 260     # return $running;
 261 }
 262
 263 # transfer all data and move VM config files
 264 sub phase1 {
 265     my ($self, $vmid) = @_;
 266     die "abstract method - implement me";
 267 }
 268
 269 # only called if there are errors in phase1
 270 sub phase1_cleanup {
 271     my ($self, $vmid, $err) = @_;
 272     die "abstract method - implement me";
 273 }
 274
 275 # only called when VM is running and phase1 was successful
 276 sub phase2 {
 277     my ($self, $vmid) = @_;
 278     die "abstract method - implement me";
 279 }
 280
 281 # only called when VM is running and phase1 was successful
 282 sub phase2_cleanup {
 283     my ($self, $vmid, $err) = @_;
 284 };
 285
 286 #  only called when phase1 was successful
 287 sub phase3 {
 288     my ($self, $vmid) = @_;
 289 }
 290
 291 #  only called when phase1 was successful
 292 sub phase3_cleanup {
 293     my ($self, $vmid, $err) = @_;
 294 }
 295
 296 # final cleanup - always called
 297 sub final_cleanup {
 298     my ($self, $vmid) = @_;
 299     die "abstract method - implement me";
 300 }
 301
 302 # transfer replication state helper - call this before moving the guest config
 303 # - just log the error is something fails
 304 sub transfer_replication_state {
 305     my ($self) = @_;
 306
 307     my $local_node = PVE::INotify::nodename();
 308
 309     eval {
 310         my $stateobj = PVE::ReplicationState::read_state();
 311         my $vmstate = PVE::ReplicationState::extract_vmid_tranfer_state($stateobj, $self->{vmid}, $self->{node}, $local_node);
 312         # This have to be quoted when it run it over ssh.
 313         my $state = PVE::Tools::shellquote(encode_json($vmstate));
 314         my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $self->{vmid}, $state];
 315         $self->cmd($cmd);
 316     };
 317     if (my $err = $@) {
 318         $self->log('err', "transfer replication state failed - $err");
 319         $self->{errors} = 1;
 320     }
 321 }
 322
 323 # switch replication job target - call this after moving the guest config
 324 # - does nothing if there is no relevant replication job
 325 # - just log the error is something fails
 326 sub switch_replication_job_target {
 327     my ($self) = @_;
 328
 329     my $local_node = PVE::INotify::nodename();
 330
 331     eval { PVE::ReplicationConfig::switch_replication_job_target($self->{vmid}, $self->{node}, $local_node); };
 332     if (my $err = $@) {
 333         $self->log('err', "switch replication job target failed - $err");
 334         $self->{errors} = 1;
 335     }
 336 }
 337
 338 1;