src/PVE/LXC/Migrate.pm

   1 package PVE::LXC::Migrate;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Copy; # fixme: remove
   8
   9 use PVE::Cluster;
  10 use PVE::INotify;
  11 use PVE::Replication;
  12 use PVE::ReplicationConfig;
  13 use PVE::ReplicationState;
  14 use PVE::Storage;
  15 use PVE::Tools;
  16
  17 use PVE::LXC::Config;
  18 use PVE::LXC;
  19
  20 use PVE::AbstractMigrate;
  21 use base qw(PVE::AbstractMigrate);
  22
  23 sub lock_vm {
  24     my ($self, $vmid, $code, @param) = @_;
  25
  26     return PVE::LXC::Config->lock_config($vmid, $code, @param);
  27 }
  28
  29 sub prepare {
  30     my ($self, $vmid) = @_;
  31
  32     my $online = $self->{opts}->{online};
  33     my $restart= $self->{opts}->{restart};
  34
  35     $self->{storecfg} = PVE::Storage::config();
  36
  37     # test if CT exists
  38     my $conf = $self->{vmconf} = PVE::LXC::Config->load_config($vmid);
  39
  40     PVE::LXC::Config->check_lock($conf);
  41
  42     my $running = 0;
  43     if (PVE::LXC::check_running($vmid)) {
  44         die "lxc live migration is currently not implemented\n" if $online;
  45         die "running container can only be migrated in restart mode" if !$restart;
  46         $running = 1;
  47     }
  48     $self->{was_running} = $running;
  49
  50     PVE::LXC::Config->foreach_volume_full($conf, { include_unused => 1 }, sub {
  51         my ($ms, $mountpoint) = @_;
  52
  53         my $volid = $mountpoint->{volume};
  54         my $type = $mountpoint->{type};
  55
  56         # skip dev/bind mps when shared
  57         if ($type ne 'volume') {
  58             if ($mountpoint->{shared}) {
  59                 return;
  60             } else {
  61                 die "cannot migrate local $type mount point '$ms'\n";
  62             }
  63         }
  64
  65         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1) if $volid;
  66         die "can't determine assigned storage for mount point '$ms'\n" if !$storage;
  67
  68         # check if storage is available on both nodes
  69         my $scfg = PVE::Storage::storage_check_enabled($self->{storecfg}, $storage);
  70
  71         my $targetsid = $storage;
  72
  73         die "content type 'rootdir' is not available on storage '$storage'\n"
  74             if !$scfg->{content}->{rootdir};
  75
  76         if ($scfg->{shared}) {
  77             # PVE::Storage::activate_storage checks this for non-shared storages
  78             my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
  79             warn "Used shared storage '$storage' is not online on source node!\n"
  80                 if !$plugin->check_connection($storage, $scfg);
  81         } else {
  82             # unless in restart mode because we shut the container down
  83             die "unable to migrate local mount point '$volid' while CT is running"
  84                 if $running && !$restart;
  85
  86             $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $storage);
  87         }
  88
  89         my $target_scfg = PVE::Storage::storage_check_enabled($self->{storecfg}, $targetsid, $self->{node});
  90
  91         die "$volid: content type 'rootdir' is not available on storage '$targetsid'\n"
  92             if !$target_scfg->{content}->{rootdir};
  93     });
  94
  95     # todo: test if VM uses local resources
  96
  97     # test ssh connection
  98     my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
  99     eval { $self->cmd_quiet($cmd); };
 100     die "Can't connect to destination address using public key\n" if $@;
 101
 102     # in restart mode, we shutdown the container before migrating
 103     if ($restart && $running) {
 104         my $timeout = $self->{opts}->{timeout} // 180;
 105
 106         $self->log('info', "shutdown CT $vmid\n");
 107
 108         PVE::LXC::vm_stop($vmid, 0, $timeout);
 109
 110         $running = 0;
 111     }
 112
 113     return $running;
 114 }
 115
 116 sub phase1 {
 117     my ($self, $vmid) = @_;
 118
 119     $self->log('info', "starting migration of CT $self->{vmid} to node '$self->{node}' ($self->{nodeip})");
 120
 121     my $conf = $self->{vmconf};
 122     $conf->{lock} = 'migrate';
 123     PVE::LXC::Config->write_config($vmid, $conf);
 124
 125     if ($self->{running}) {
 126         $self->log('info', "container is running - using online migration");
 127     }
 128
 129     $self->{volumes} = []; # list of already migrated volumes
 130     my $volhash = {}; # 'config', 'snapshot' or 'storage' for local volumes
 131     my $volhash_errors = {};
 132     my $abort = 0;
 133
 134     my $log_error = sub {
 135         my ($msg, $volid) = @_;
 136
 137         $volhash_errors->{$volid} = $msg if !defined($volhash_errors->{$volid});
 138         $abort = 1;
 139     };
 140
 141     my $test_volid = sub {
 142         my ($volid, $snapname) = @_;
 143
 144         return if !$volid;
 145
 146         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 147
 148         # check if storage is available on source node
 149         my $scfg = PVE::Storage::storage_check_enabled($self->{storecfg}, $sid);
 150
 151         my $targetsid = $sid;
 152
 153         if ($scfg->{shared}) {
 154             $self->log('info', "volume '$volid' is on shared storage '$sid'")
 155                 if !$snapname;
 156             return;
 157         } else {
 158             $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid);
 159         }
 160
 161         PVE::Storage::storage_check_enabled($self->{storecfg}, $targetsid, $self->{node});
 162
 163         my $bwlimit = $self->get_bwlimit($sid, $targetsid);
 164
 165         $volhash->{$volid}->{ref} = defined($snapname) ? 'snapshot' : 'config';
 166         $volhash->{$volid}->{snapshots} = 1 if defined($snapname);
 167         $volhash->{$volid}->{targetsid} = $targetsid;
 168         $volhash->{$volid}->{bwlimit} = $bwlimit;
 169
 170         my ($path, $owner) = PVE::Storage::path($self->{storecfg}, $volid);
 171
 172         die "owned by other guest (owner = $owner)\n"
 173             if !$owner || ($owner != $self->{vmid});
 174
 175         if (defined($snapname)) {
 176             # we cannot migrate shapshots on local storage
 177             # exceptions: 'zfspool', 'btrfs'
 178             if ($scfg->{type} eq 'zfspool' || $scfg->{type} eq 'btrfs') {
 179                 return;
 180             }
 181             die "non-migratable snapshot exists\n";
 182         }
 183     };
 184
 185     my $test_mp = sub {
 186         my ($ms, $mountpoint, $snapname) = @_;
 187
 188         my $volid = $mountpoint->{volume};
 189         # already checked in prepare
 190         if ($mountpoint->{type} ne 'volume') {
 191             $self->log('info', "ignoring shared '$mountpoint->{type}' mount point '$ms' ('$volid')")
 192                 if !$snapname;
 193             return;
 194         }
 195
 196         eval {
 197             &$test_volid($volid, $snapname);
 198         };
 199
 200         &$log_error($@, $volid) if $@;
 201     };
 202
 203     # first unused / lost volumes owned by this container
 204     my @sids = PVE::Storage::storage_ids($self->{storecfg});
 205     foreach my $storeid (@sids) {
 206         my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
 207         next if $scfg->{shared};
 208         next if !PVE::Storage::storage_check_enabled($self->{storecfg}, $storeid, undef, 1);
 209
 210         # get list from PVE::Storage (for unreferenced volumes)
 211         my $dl = PVE::Storage::vdisk_list($self->{storecfg}, $storeid, $vmid, undef, 'rootdir');
 212
 213         next if @{$dl->{$storeid}} == 0;
 214
 215         # check if storage is available on target node
 216         my $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $storeid);
 217         my $target_scfg = PVE::Storage::storage_check_enabled($self->{storecfg}, $targetsid, $self->{node});
 218
 219         die "content type 'rootdir' is not available on storage '$targetsid'\n"
 220             if !$target_scfg->{content}->{rootdir};
 221
 222         PVE::Storage::foreach_volid($dl, sub {
 223             my ($volid, $sid, $volname) = @_;
 224
 225             $volhash->{$volid}->{ref} = 'storage';
 226             $volhash->{$volid}->{targetsid} = $targetsid;
 227         });
 228     }
 229
 230     # then all volumes referenced in snapshots
 231     foreach my $snapname (keys %{$conf->{snapshots}}) {
 232         &$test_volid($conf->{snapshots}->{$snapname}->{'vmstate'}, 0, undef)
 233             if defined($conf->{snapshots}->{$snapname}->{'vmstate'});
 234         PVE::LXC::Config->foreach_volume($conf->{snapshots}->{$snapname}, $test_mp, $snapname);
 235     }
 236
 237     # finally all current volumes
 238     PVE::LXC::Config->foreach_volume_full($conf, { include_unused => 1 }, $test_mp);
 239
 240     # additional checks for local storage
 241     foreach my $volid (keys %$volhash) {
 242         eval {
 243             my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 244             my $scfg =  PVE::Storage::storage_config($self->{storecfg}, $sid);
 245
 246             my $migratable = ($scfg->{type} eq 'dir') || ($scfg->{type} eq 'zfspool')
 247                 || ($scfg->{type} eq 'lvmthin') || ($scfg->{type} eq 'lvm')
 248                 || ($scfg->{type} eq 'btrfs');
 249
 250             die "storage type '$scfg->{type}' not supported\n"
 251                 if !$migratable;
 252
 253             # image is a linked clone on local storage, se we can't migrate.
 254             if (my $basename = (PVE::Storage::parse_volname($self->{storecfg}, $volid))[3]) {
 255                 die "clone of '$basename'";
 256             }
 257         };
 258         &$log_error($@, $volid) if $@;
 259     }
 260
 261     foreach my $volid (sort keys %$volhash) {
 262         my $ref = $volhash->{$volid}->{ref};
 263         if ($ref eq 'storage') {
 264             $self->log('info', "found local volume '$volid' (via storage)\n");
 265         } elsif ($ref eq 'config') {
 266             $self->log('info', "found local volume '$volid' (in current VM config)\n");
 267         } elsif ($ref eq 'snapshot') {
 268             $self->log('info', "found local volume '$volid' (referenced by snapshot(s))\n");
 269         } else {
 270             $self->log('info', "found local volume '$volid'\n");
 271         }
 272     }
 273
 274     foreach my $volid (sort keys %$volhash_errors) {
 275         $self->log('warn', "can't migrate local volume '$volid': $volhash_errors->{$volid}");
 276     }
 277
 278     if ($abort) {
 279         die "can't migrate CT - check log\n";
 280     }
 281
 282     my $rep_volumes;
 283
 284     my $rep_cfg = PVE::ReplicationConfig->new();
 285
 286     if (my $jobcfg = $rep_cfg->find_local_replication_job($vmid, $self->{node})) {
 287         die "can't live migrate VM with replicated volumes\n" if $self->{running};
 288         my $start_time = time();
 289         my $logfunc = sub { my ($msg) = @_;  $self->log('info', $msg); };
 290         $rep_volumes = PVE::Replication::run_replication(
 291             'PVE::LXC::Config', $jobcfg, $start_time, $start_time, $logfunc);
 292     }
 293
 294     my $opts = $self->{opts};
 295     foreach my $volid (keys %$volhash) {
 296         next if $rep_volumes->{$volid};
 297         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 298         push @{$self->{volumes}}, $volid;
 299
 300         # JSONSchema and get_bandwidth_limit use kbps - storage_migrate bps
 301         my $bwlimit = $volhash->{$volid}->{bwlimit};
 302         $bwlimit = $bwlimit * 1024 if defined($bwlimit);
 303
 304         my $targetsid = $volhash->{$volid}->{targetsid};
 305
 306         my $new_volid = eval {
 307             my $storage_migrate_opts = {
 308                 'ratelimit_bps' => $bwlimit,
 309                 'insecure' => $opts->{migration_type} eq 'insecure',
 310                 'with_snapshots' => $volhash->{$volid}->{snapshots},
 311                 'allow_rename' => 1,
 312             };
 313
 314             my $logfunc = sub { $self->log('info', $_[0]); };
 315             return PVE::Storage::storage_migrate(
 316                 $self->{storecfg},
 317                 $volid,
 318                 $self->{ssh_info},
 319                 $targetsid,
 320                 $storage_migrate_opts,
 321                 $logfunc,
 322             );
 323         };
 324
 325         if (my $err = $@) {
 326             die "storage migration for '$volid' to storage '$targetsid' failed - $err\n";
 327         }
 328
 329         $self->{volume_map}->{$volid} = $new_volid;
 330         $self->log('info', "volume '$volid' is '$new_volid' on the target\n");
 331
 332         eval { PVE::Storage::deactivate_volumes($self->{storecfg}, [$volid]); };
 333         if (my $err = $@) {
 334             $self->log('warn', $err);
 335         }
 336     }
 337
 338     if ($self->{running}) {
 339         die "implement me";
 340     }
 341
 342     # make sure everything on (shared) storage is unmounted
 343     # Note: we must be 100% sure, else we get data corruption because
 344     # non-shared file system could be mounted twice (on shared storage)
 345
 346     PVE::LXC::umount_all($vmid, $self->{storecfg}, $conf);
 347
 348     #to be sure there are no active volumes
 349     my $vollist = PVE::LXC::Config->get_vm_volumes($conf);
 350     PVE::Storage::deactivate_volumes($self->{storecfg}, $vollist);
 351
 352     # transfer replication state before moving config
 353     $self->transfer_replication_state() if $rep_volumes;
 354     PVE::LXC::Config->update_volume_ids($conf, $self->{volume_map});
 355     PVE::LXC::Config->write_config($vmid, $conf);
 356     PVE::LXC::Config->move_config_to_node($vmid, $self->{node});
 357     $self->{conf_migrated} = 1;
 358     $self->switch_replication_job_target() if $rep_volumes;
 359 }
 360
 361 sub phase1_cleanup {
 362     my ($self, $vmid, $err) = @_;
 363
 364     $self->log('info', "aborting phase 1 - cleanup resources");
 365
 366     if ($self->{volumes}) {
 367         foreach my $volid (@{$self->{volumes}}) {
 368             $self->log('err', "found stale volume copy '$volid' on node '$self->{node}'");
 369             # fixme: try to remove ?
 370         }
 371     }
 372 }
 373
 374 sub phase3 {
 375     my ($self, $vmid) = @_;
 376
 377     my $volids = $self->{volumes};
 378
 379     # destroy local copies
 380     foreach my $volid (@$volids) {
 381         eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
 382         if (my $err = $@) {
 383             $self->log('err', "removing local copy of '$volid' failed - $err");
 384             $self->{errors} = 1;
 385             last if $err =~ /^interrupted by signal$/;
 386         }
 387     }
 388 }
 389
 390 sub final_cleanup {
 391     my ($self, $vmid) = @_;
 392
 393     $self->log('info', "start final cleanup");
 394
 395     if (!$self->{conf_migrated}) {
 396         eval { PVE::LXC::Config->remove_lock($vmid, 'migrate'); };
 397         if (my $err = $@) {
 398             $self->log('err', $err);
 399         }
 400         # in restart mode, we start the container on the source node
 401         # on migration error
 402         if ($self->{opts}->{restart} && $self->{was_running}) {
 403             $self->log('info', "start container on source node");
 404             my $skiplock = 1;
 405             PVE::LXC::vm_start($vmid, $self->{vmconf}, $skiplock);
 406         }
 407     } else {
 408         my $cmd = [ @{$self->{rem_ssh}}, 'pct', 'unlock', $vmid ];
 409         $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
 410
 411         # in restart mode, we start the container on the target node
 412         # after migration
 413         if ($self->{opts}->{restart} && $self->{was_running}) {
 414             $self->log('info', "start container on target node");
 415             my $cmd = [ @{$self->{rem_ssh}}, 'pct', 'start', $vmid];
 416             $self->cmd($cmd);
 417         }
 418     }
 419
 420 }
 421
 422 1;