PVE/AbstractMigrate.pm

   1 package PVE::AbstractMigrate;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(strftime);
   6 use JSON;
   7 use PVE::Tools;
   8 use PVE::Cluster;
   9 use PVE::DataCenterConfig;
  10 use PVE::ReplicationState;
  11
  12 my $msg2text = sub {
  13     my ($level, $msg) = @_;
  14
  15     chomp $msg;
  16
  17     return '' if !$msg;
  18
  19     my $res = '';
  20
  21     my $tstr = strftime("%F %H:%M:%S", localtime);
  22
  23     foreach my $line (split (/\n/, $msg)) {
  24         if ($level eq 'err') {
  25             $res .= "$tstr ERROR: $line\n";
  26         } else {
  27             $res .= "$tstr $line\n";
  28         }
  29     }
  30
  31     return $res;
  32 };
  33
  34 sub log {
  35     my ($self, $level, $msg) = @_;
  36
  37     chomp $msg;
  38
  39     return if !$msg;
  40
  41     print &$msg2text($level, $msg);
  42 }
  43
  44 sub cmd {
  45     my ($self, $cmd, %param) = @_;
  46
  47     my $logfunc = sub {
  48         my $line = shift;
  49         $self->log('info', $line);
  50     };
  51
  52     $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  53
  54     PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc);
  55 }
  56
  57 my $run_command_quiet_full = sub {
  58     my ($self, $cmd, $logerr, %param) = @_;
  59
  60     my $log = '';
  61     my $logfunc = sub {
  62         my $line = shift;
  63         $log .= &$msg2text('info', $line);;
  64     };
  65
  66     eval { PVE::Tools::run_command($cmd, %param, outfunc => $logfunc, errfunc => $logfunc); };
  67     if (my $err = $@) {
  68         $self->log('info', "# " . PVE::Tools::cmd2string($cmd));
  69         print $log;
  70         if ($logerr) {
  71             $self->{errors} = 1;
  72             $self->log('err', $err);
  73         } else {
  74             die $err;
  75         }
  76     }
  77 };
  78
  79 sub cmd_quiet {
  80     my ($self, $cmd, %param) = @_;
  81     return &$run_command_quiet_full($self, $cmd, 0, %param);
  82 }
  83
  84 sub cmd_logerr {
  85     my ($self, $cmd, %param) = @_;
  86     return &$run_command_quiet_full($self, $cmd, 1, %param);
  87 }
  88
  89 my $eval_int = sub {
  90     my ($self, $func, @param) = @_;
  91
  92     eval {
  93         local $SIG{INT} =
  94             local $SIG{TERM} =
  95             local $SIG{QUIT} =
  96             local $SIG{HUP} =
  97             local $SIG{PIPE} = sub {
  98                 $self->{delayed_interrupt} = 0;
  99                 die "interrupted by signal\n";
 100         };
 101
 102         my $di = $self->{delayed_interrupt};
 103         $self->{delayed_interrupt} = 0;
 104
 105         die "interrupted by signal\n" if $di;
 106
 107         &$func($self, @param);
 108     };
 109 };
 110
 111 # FIXME: nodeip is now unused
 112 sub migrate {
 113     my ($class, $node, $nodeip, $vmid, $opts) = @_;
 114
 115     $class = ref($class) || $class;
 116
 117     my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 118
 119     my $migration_network = $opts->{migration_network};
 120     if (!defined($migration_network)) {
 121         $migration_network = $dc_conf->{migration}->{network};
 122     }
 123     my $ssh_info = PVE::Cluster::get_ssh_info($node, $migration_network);
 124     $nodeip = $ssh_info->{ip};
 125
 126     my $migration_type = 'secure';
 127     if (defined($opts->{migration_type})) {
 128         $migration_type = $opts->{migration_type};
 129     } elsif (defined($dc_conf->{migration}->{type})) {
 130         $migration_type = $dc_conf->{migration}->{type};
 131     }
 132     $opts->{migration_type} = $migration_type;
 133
 134     my $self = {
 135         delayed_interrupt => 0,
 136         opts => $opts,
 137         vmid => $vmid,
 138         node => $node,
 139         ssh_info => $ssh_info,
 140         nodeip => $nodeip,
 141         rem_ssh => PVE::Cluster::ssh_info_to_command($ssh_info)
 142     };
 143
 144     $self = bless $self, $class;
 145
 146     my $starttime = time();
 147
 148     local $SIG{INT} =
 149         local $SIG{TERM} =
 150         local $SIG{QUIT} =
 151         local $SIG{HUP} =
 152         local $SIG{PIPE} = sub {
 153             $self->log('err', "received interrupt - delayed");
 154             $self->{delayed_interrupt} = 1;
 155     };
 156
 157     # lock container during migration
 158     eval { $self->lock_vm($self->{vmid}, sub {
 159
 160         $self->{running} = 0;
 161         &$eval_int($self, sub { $self->{running} = $self->prepare($self->{vmid}); });
 162         die $@ if $@;
 163
 164         if (defined($migration_network)) {
 165             $self->log('info', "use dedicated network address for sending " .
 166                        "migration traffic ($self->{nodeip})");
 167
 168             # test if we can connect to new IP
 169             my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
 170             eval { $self->cmd_quiet($cmd); };
 171             die "Can't connect to destination address ($self->{nodeip}) using " .
 172                 "public key authentication\n" if $@;
 173         }
 174
 175         &$eval_int($self, sub { $self->phase1($self->{vmid}); });
 176         my $err = $@;
 177         if ($err) {
 178             $self->log('err', $err);
 179             eval { $self->phase1_cleanup($self->{vmid}, $err); };
 180             if (my $tmperr = $@) {
 181                 $self->log('err', $tmperr);
 182             }
 183             eval { $self->final_cleanup($self->{vmid}); };
 184             if (my $tmperr = $@) {
 185                 $self->log('err', $tmperr);
 186             }
 187             die $err;
 188         }
 189
 190         # vm is now owned by other node
 191         # Note: there is no VM config file on the local node anymore
 192
 193         if ($self->{running}) {
 194
 195             &$eval_int($self, sub { $self->phase2($self->{vmid}); });
 196             my $phase2err = $@;
 197             if ($phase2err) {
 198                 $self->{errors} = 1;
 199                 $self->log('err', "online migrate failure - $phase2err");
 200             }
 201             eval { $self->phase2_cleanup($self->{vmid}, $phase2err); };
 202             if (my $err = $@) {
 203                 $self->log('err', $err);
 204                 $self->{errors} = 1;
 205             }
 206         }
 207
 208         # phase3 (finalize)
 209         &$eval_int($self, sub { $self->phase3($self->{vmid}); });
 210         my $phase3err = $@;
 211         if ($phase3err) {
 212             $self->log('err', $phase3err);
 213             $self->{errors} = 1;
 214         }
 215         eval { $self->phase3_cleanup($self->{vmid}, $phase3err); };
 216         if (my $err = $@) {
 217             $self->log('err', $err);
 218             $self->{errors} = 1;
 219         }
 220         eval { $self->final_cleanup($self->{vmid}); };
 221         if (my $err = $@) {
 222             $self->log('err', $err);
 223             $self->{errors} = 1;
 224         }
 225     })};
 226
 227     my $err = $@;
 228
 229     my $delay = time() - $starttime;
 230     my $mins = int($delay/60);
 231     my $secs = $delay - $mins*60;
 232     my $hours =  int($mins/60);
 233     $mins = $mins - $hours*60;
 234
 235     my $duration = sprintf "%02d:%02d:%02d", $hours, $mins, $secs;
 236
 237     if ($err) {
 238         $self->log('err', "migration aborted (duration $duration): $err");
 239         die "migration aborted\n";
 240     }
 241
 242     if ($self->{errors}) {
 243         $self->log('err', "migration finished with problems (duration $duration)");
 244         die "migration problems\n"
 245     }
 246
 247     $self->log('info', "migration finished successfully (duration $duration)");
 248 }
 249
 250 sub lock_vm {
 251     my ($self, $vmid, $code, @param) = @_;
 252
 253     die "abstract method - implement me";
 254 }
 255
 256 sub prepare {
 257     my ($self, $vmid) = @_;
 258
 259     die "abstract method - implement me";
 260
 261     # return $running;
 262 }
 263
 264 # transfer all data and move VM config files
 265 sub phase1 {
 266     my ($self, $vmid) = @_;
 267     die "abstract method - implement me";
 268 }
 269
 270 # only called if there are errors in phase1
 271 sub phase1_cleanup {
 272     my ($self, $vmid, $err) = @_;
 273     die "abstract method - implement me";
 274 }
 275
 276 # only called when VM is running and phase1 was successful
 277 sub phase2 {
 278     my ($self, $vmid) = @_;
 279     die "abstract method - implement me";
 280 }
 281
 282 # only called when VM is running and phase1 was successful
 283 sub phase2_cleanup {
 284     my ($self, $vmid, $err) = @_;
 285 };
 286
 287 #  only called when phase1 was successful
 288 sub phase3 {
 289     my ($self, $vmid) = @_;
 290 }
 291
 292 #  only called when phase1 was successful
 293 sub phase3_cleanup {
 294     my ($self, $vmid, $err) = @_;
 295 }
 296
 297 # final cleanup - always called
 298 sub final_cleanup {
 299     my ($self, $vmid) = @_;
 300     die "abstract method - implement me";
 301 }
 302
 303 # transfer replication state helper - call this before moving the guest config
 304 # - just log the error is something fails
 305 sub transfer_replication_state {
 306     my ($self) = @_;
 307
 308     my $local_node = PVE::INotify::nodename();
 309
 310     eval {
 311         my $stateobj = PVE::ReplicationState::read_state();
 312         my $vmstate = PVE::ReplicationState::extract_vmid_tranfer_state($stateobj, $self->{vmid}, $self->{node}, $local_node);
 313         # This have to be quoted when it run it over ssh.
 314         my $state = PVE::Tools::shellquote(encode_json($vmstate));
 315         my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $self->{vmid}, $state];
 316         $self->cmd($cmd);
 317     };
 318     if (my $err = $@) {
 319         $self->log('err', "transfer replication state failed - $err");
 320         $self->{errors} = 1;
 321     }
 322 }
 323
 324 # switch replication job target - call this after moving the guest config
 325 # - does nothing if there is no relevant replication job
 326 # - just log the error is something fails
 327 sub switch_replication_job_target {
 328     my ($self) = @_;
 329
 330     my $local_node = PVE::INotify::nodename();
 331
 332     eval { PVE::ReplicationConfig::switch_replication_job_target($self->{vmid}, $self->{node}, $local_node); };
 333     if (my $err = $@) {
 334         $self->log('err', "switch replication job target failed - $err");
 335         $self->{errors} = 1;
 336     }
 337 }
 338
 339 1;