src/PVE/LXC/Migrate.pm

   1 package PVE::LXC::Migrate;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Copy; # fixme: remove
   8
   9 use PVE::Cluster;
  10 use PVE::INotify;
  11 use PVE::Replication;
  12 use PVE::ReplicationConfig;
  13 use PVE::ReplicationState;
  14 use PVE::Storage;
  15 use PVE::Tools;
  16
  17 use PVE::LXC::Config;
  18 use PVE::LXC;
  19
  20 use PVE::AbstractMigrate;
  21 use base qw(PVE::AbstractMigrate);
  22
  23 # compared against remote end's minimum version
  24 our $WS_TUNNEL_VERSION = 2;
  25
  26 sub lock_vm {
  27     my ($self, $vmid, $code, @param) = @_;
  28
  29     return PVE::LXC::Config->lock_config($vmid, $code, @param);
  30 }
  31
  32 sub prepare {
  33     my ($self, $vmid) = @_;
  34
  35     my $online = $self->{opts}->{online};
  36     my $restart= $self->{opts}->{restart};
  37     my $remote = $self->{opts}->{remote};
  38
  39     $self->{storecfg} = PVE::Storage::config();
  40
  41     # test if CT exists
  42     my $conf = $self->{vmconf} = PVE::LXC::Config->load_config($vmid);
  43
  44     PVE::LXC::Config->check_lock($conf);
  45
  46     my $running = 0;
  47     if (PVE::LXC::check_running($vmid)) {
  48         die "lxc live migration is currently not implemented\n" if $online;
  49         die "running container can only be migrated in restart mode" if !$restart;
  50         $running = 1;
  51     }
  52     $self->{was_running} = $running;
  53
  54     my $storages = {};
  55     PVE::LXC::Config->foreach_volume_full($conf, { include_unused => 1 }, sub {
  56         my ($ms, $mountpoint) = @_;
  57
  58         my $volid = $mountpoint->{volume};
  59         my $type = $mountpoint->{type};
  60
  61         # skip dev/bind mps when shared
  62         if ($type ne 'volume') {
  63             if ($mountpoint->{shared}) {
  64                 return;
  65             } else {
  66                 die "cannot migrate local $type mount point '$ms'\n";
  67             }
  68         }
  69
  70         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1) if $volid;
  71         die "can't determine assigned storage for mount point '$ms'\n" if !$storage;
  72
  73         # check if storage is available on both nodes
  74         my $scfg = PVE::Storage::storage_check_enabled($self->{storecfg}, $storage);
  75
  76         my $targetsid = $storage;
  77
  78         die "content type 'rootdir' is not available on storage '$storage'\n"
  79             if !$scfg->{content}->{rootdir};
  80
  81         if ($scfg->{shared} && !$remote) {
  82             # PVE::Storage::activate_storage checks this for non-shared storages
  83             my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
  84             warn "Used shared storage '$storage' is not online on source node!\n"
  85                 if !$plugin->check_connection($storage, $scfg);
  86         } else {
  87             # unless in restart mode because we shut the container down
  88             die "unable to migrate local mount point '$volid' while CT is running"
  89                 if $running && !$restart;
  90
  91             $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $storage);
  92         }
  93
  94         if (!$remote) {
  95             my $target_scfg = PVE::Storage::storage_check_enabled($self->{storecfg}, $targetsid, $self->{node});
  96
  97             die "$volid: content type 'rootdir' is not available on storage '$targetsid'\n"
  98                 if !$target_scfg->{content}->{rootdir};
  99         }
 100
 101         $storages->{$targetsid} = 1;
 102     });
 103
 104     # todo: test if VM uses local resources
 105
 106     if ($remote) {
 107         # test & establish websocket connection
 108         my $bridges = map_bridges($conf, $self->{opts}->{bridgemap}, 1);
 109
 110         my $remote = $self->{opts}->{remote};
 111         my $conn = $remote->{conn};
 112
 113         my $log = sub {
 114             my ($level, $msg) = @_;
 115             $self->log($level, $msg);
 116         };
 117
 118         my $websocket_url = "https://$conn->{host}:$conn->{port}/api2/json/nodes/$self->{node}/lxc/$remote->{vmid}/mtunnelwebsocket";
 119         my $url = "/nodes/$self->{node}/lxc/$remote->{vmid}/mtunnel";
 120
 121         my $tunnel_params = {
 122             url => $websocket_url,
 123         };
 124
 125         my $storage_list = join(',', keys %$storages);
 126         my $bridge_list = join(',', keys %$bridges);
 127
 128         my $req_params = {
 129             storages => $storage_list,
 130             bridges => $bridge_list,
 131         };
 132
 133         my $tunnel = PVE::Tunnel::fork_websocket_tunnel($conn, $url, $req_params, $tunnel_params, $log);
 134         my $min_version = $tunnel->{version} - $tunnel->{age};
 135         $self->log('info', "local WS tunnel version: $WS_TUNNEL_VERSION");
 136         $self->log('info', "remote WS tunnel version: $tunnel->{version}");
 137         $self->log('info', "minimum required WS tunnel version: $min_version");
 138         die "Remote tunnel endpoint not compatible, upgrade required\n"
 139             if $WS_TUNNEL_VERSION < $min_version;
 140          die "Remote tunnel endpoint too old, upgrade required\n"
 141             if $WS_TUNNEL_VERSION > $tunnel->{version};
 142
 143         $self->log('info', "websocket tunnel started\n");
 144         $self->{tunnel} = $tunnel;
 145     } else {
 146         # test ssh connection
 147         my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
 148         eval { $self->cmd_quiet($cmd); };
 149         die "Can't connect to destination address using public key\n" if $@;
 150     }
 151
 152     # in restart mode, we shutdown the container before migrating
 153     if ($restart && $running) {
 154         my $timeout = $self->{opts}->{timeout} // 180;
 155
 156         $self->log('info', "shutdown CT $vmid\n");
 157
 158         PVE::LXC::vm_stop($vmid, 0, $timeout);
 159
 160         $running = 0;
 161     }
 162
 163     return $running;
 164 }
 165
 166 sub phase1 {
 167     my ($self, $vmid) = @_;
 168
 169     my $remote = $self->{opts}->{remote};
 170
 171     $self->log('info', "starting migration of CT $self->{vmid} to node '$self->{node}' ($self->{nodeip})");
 172
 173     my $conf = $self->{vmconf};
 174     $conf->{lock} = 'migrate';
 175     PVE::LXC::Config->write_config($vmid, $conf);
 176
 177     if ($self->{running}) {
 178         $self->log('info', "container is running - using online migration");
 179     }
 180
 181     $self->{volumes} = []; # list of already migrated volumes
 182     my $volhash = {}; # 'config', 'snapshot' or 'storage' for local volumes
 183     my $volhash_errors = {};
 184     my $abort = 0;
 185
 186     my $log_error = sub {
 187         my ($msg, $volid) = @_;
 188
 189         $volhash_errors->{$volid} = $msg if !defined($volhash_errors->{$volid});
 190         $abort = 1;
 191     };
 192
 193     my $test_volid = sub {
 194         my ($volid, $snapname) = @_;
 195
 196         return if !$volid;
 197
 198         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 199
 200         # check if storage is available on source node
 201         my $scfg = PVE::Storage::storage_check_enabled($self->{storecfg}, $sid);
 202
 203         my $targetsid = $sid;
 204
 205         if ($scfg->{shared} && !$remote) {
 206             $self->log('info', "volume '$volid' is on shared storage '$sid'")
 207                 if !$snapname;
 208             return;
 209         } else {
 210             $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid);
 211         }
 212
 213         PVE::Storage::storage_check_enabled($self->{storecfg}, $targetsid, $self->{node})
 214             if !$remote;
 215
 216         my $bwlimit = $self->get_bwlimit($sid, $targetsid);
 217
 218         $volhash->{$volid}->{ref} = defined($snapname) ? 'snapshot' : 'config';
 219         $volhash->{$volid}->{snapshots} = 1 if defined($snapname);
 220         $volhash->{$volid}->{targetsid} = $targetsid;
 221         $volhash->{$volid}->{bwlimit} = $bwlimit;
 222
 223         my ($path, $owner) = PVE::Storage::path($self->{storecfg}, $volid);
 224
 225         die "owned by other guest (owner = $owner)\n"
 226             if !$owner || ($owner != $self->{vmid});
 227
 228         if (defined($snapname)) {
 229             # we cannot migrate shapshots on local storage
 230             # exceptions: 'zfspool', 'btrfs'
 231             if ($scfg->{type} eq 'zfspool' || $scfg->{type} eq 'btrfs') {
 232                 return;
 233             }
 234             die "non-migratable snapshot exists\n";
 235         }
 236     };
 237
 238     my $test_mp = sub {
 239         my ($ms, $mountpoint, $snapname) = @_;
 240
 241         my $volid = $mountpoint->{volume};
 242         # already checked in prepare
 243         if ($mountpoint->{type} ne 'volume') {
 244             $self->log('info', "ignoring shared '$mountpoint->{type}' mount point '$ms' ('$volid')")
 245                 if !$snapname;
 246             return;
 247         }
 248
 249         eval {
 250             &$test_volid($volid, $snapname);
 251
 252             die "remote migration with snapshots not supported yet\n"
 253                 if $remote && $snapname;
 254         };
 255
 256         &$log_error($@, $volid) if $@;
 257     };
 258
 259     # first unused / lost volumes owned by this container
 260     my @sids = PVE::Storage::storage_ids($self->{storecfg});
 261     foreach my $storeid (@sids) {
 262         my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
 263         next if $scfg->{shared} && !$remote;
 264         next if !PVE::Storage::storage_check_enabled($self->{storecfg}, $storeid, undef, 1);
 265
 266         # get list from PVE::Storage (for unreferenced volumes)
 267         my $dl = PVE::Storage::vdisk_list($self->{storecfg}, $storeid, $vmid, undef, 'rootdir');
 268
 269         next if @{$dl->{$storeid}} == 0;
 270
 271         # check if storage is available on target node
 272         my $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $storeid);
 273         if (!$remote) {
 274             my $target_scfg = PVE::Storage::storage_check_enabled($self->{storecfg}, $targetsid, $self->{node});
 275
 276             die "content type 'rootdir' is not available on storage '$targetsid'\n"
 277                 if !$target_scfg->{content}->{rootdir};
 278         }
 279
 280         PVE::Storage::foreach_volid($dl, sub {
 281             my ($volid, $sid, $volname) = @_;
 282
 283             $volhash->{$volid}->{ref} = 'storage';
 284             $volhash->{$volid}->{targetsid} = $targetsid;
 285         });
 286     }
 287
 288     # then all volumes referenced in snapshots
 289     foreach my $snapname (keys %{$conf->{snapshots}}) {
 290         &$test_volid($conf->{snapshots}->{$snapname}->{'vmstate'}, 0, undef)
 291             if defined($conf->{snapshots}->{$snapname}->{'vmstate'});
 292         PVE::LXC::Config->foreach_volume($conf->{snapshots}->{$snapname}, $test_mp, $snapname);
 293     }
 294
 295     # finally all current volumes
 296     PVE::LXC::Config->foreach_volume_full($conf, { include_unused => 1 }, $test_mp);
 297
 298     # additional checks for local storage
 299     foreach my $volid (keys %$volhash) {
 300         eval {
 301             my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 302             my $scfg =  PVE::Storage::storage_config($self->{storecfg}, $sid);
 303
 304             # TODO move to storage plugin layer?
 305             my $migratable_storages = [
 306                 'dir',
 307                 'zfspool',
 308                 'lvmthin',
 309                 'lvm',
 310                 'btrfs',
 311             ];
 312             if ($remote) {
 313                 push @$migratable_storages, 'cifs';
 314                 push @$migratable_storages, 'nfs';
 315             }
 316
 317             die "storage type '$scfg->{type}' not supported\n"
 318                 if !grep { $_ eq $scfg->{type} } @$migratable_storages;
 319
 320             # image is a linked clone on local storage, se we can't migrate.
 321             if (my $basename = (PVE::Storage::parse_volname($self->{storecfg}, $volid))[3]) {
 322                 die "clone of '$basename'";
 323             }
 324         };
 325         &$log_error($@, $volid) if $@;
 326     }
 327
 328     foreach my $volid (sort keys %$volhash) {
 329         my $ref = $volhash->{$volid}->{ref};
 330         if ($ref eq 'storage') {
 331             $self->log('info', "found local volume '$volid' (via storage)\n");
 332         } elsif ($ref eq 'config') {
 333             $self->log('info', "found local volume '$volid' (in current VM config)\n");
 334         } elsif ($ref eq 'snapshot') {
 335             $self->log('info', "found local volume '$volid' (referenced by snapshot(s))\n");
 336         } else {
 337             $self->log('info', "found local volume '$volid'\n");
 338         }
 339     }
 340
 341     foreach my $volid (sort keys %$volhash_errors) {
 342         $self->log('warn', "can't migrate local volume '$volid': $volhash_errors->{$volid}");
 343     }
 344
 345     if ($abort) {
 346         die "can't migrate CT - check log\n";
 347     }
 348
 349     my $rep_volumes;
 350
 351     my $rep_cfg = PVE::ReplicationConfig->new();
 352
 353     if ($remote) {
 354         die "cannot remote-migrate replicated VM\n"
 355             if $rep_cfg->check_for_existing_jobs($vmid, 1);
 356     } elsif (my $jobcfg = $rep_cfg->find_local_replication_job($vmid, $self->{node})) {
 357         die "can't live migrate VM with replicated volumes\n" if $self->{running};
 358         my $start_time = time();
 359         my $logfunc = sub { my ($msg) = @_;  $self->log('info', $msg); };
 360         $rep_volumes = PVE::Replication::run_replication(
 361             'PVE::LXC::Config', $jobcfg, $start_time, $start_time, $logfunc);
 362     }
 363
 364     my $opts = $self->{opts};
 365     foreach my $volid (keys %$volhash) {
 366         next if $rep_volumes->{$volid};
 367         push @{$self->{volumes}}, $volid;
 368
 369         # JSONSchema and get_bandwidth_limit use kbps - storage_migrate bps
 370         my $bwlimit = $volhash->{$volid}->{bwlimit};
 371         $bwlimit = $bwlimit * 1024 if defined($bwlimit);
 372
 373         my $targetsid = $volhash->{$volid}->{targetsid};
 374
 375         my $new_volid = eval {
 376             if ($remote) {
 377                 my $log = sub {
 378                     my ($level, $msg) = @_;
 379                     $self->log($level, $msg);
 380                 };
 381
 382                 return PVE::StorageTunnel::storage_migrate(
 383                     $self->{tunnel},
 384                     $self->{storecfg},
 385                     $volid,
 386                     $self->{vmid},
 387                     $remote->{vmid},
 388                     $volhash->{$volid},
 389                     $log,
 390                 );
 391             } else {
 392                 my $storage_migrate_opts = {
 393                     'ratelimit_bps' => $bwlimit,
 394                     'insecure' => $opts->{migration_type} eq 'insecure',
 395                     'with_snapshots' => $volhash->{$volid}->{snapshots},
 396                     'allow_rename' => 1,
 397                 };
 398
 399                 my $logfunc = sub { $self->log('info', $_[0]); };
 400                 return PVE::Storage::storage_migrate(
 401                     $self->{storecfg},
 402                     $volid,
 403                     $self->{ssh_info},
 404                     $targetsid,
 405                     $storage_migrate_opts,
 406                     $logfunc,
 407                 );
 408             }
 409         };
 410
 411         if (my $err = $@) {
 412             die "storage migration for '$volid' to storage '$targetsid' failed - $err\n";
 413         }
 414
 415         $self->{volume_map}->{$volid} = $new_volid;
 416         $self->log('info', "volume '$volid' is '$new_volid' on the target\n");
 417
 418         eval { PVE::Storage::deactivate_volumes($self->{storecfg}, [$volid]); };
 419         if (my $err = $@) {
 420             $self->log('warn', $err);
 421         }
 422     }
 423
 424     if ($self->{running}) {
 425         die "implement me";
 426     }
 427
 428     # make sure everything on (shared) storage is unmounted
 429     # Note: we must be 100% sure, else we get data corruption because
 430     # non-shared file system could be mounted twice (on shared storage)
 431
 432     PVE::LXC::umount_all($vmid, $self->{storecfg}, $conf);
 433
 434     #to be sure there are no active volumes
 435     my $vollist = PVE::LXC::Config->get_vm_volumes($conf);
 436     PVE::Storage::deactivate_volumes($self->{storecfg}, $vollist);
 437
 438     if ($remote) {
 439         my $remote_conf = PVE::LXC::Config->load_config($vmid);
 440         PVE::LXC::Config->update_volume_ids($remote_conf, $self->{volume_map});
 441
 442         my $bridges = map_bridges($remote_conf, $self->{opts}->{bridgemap});
 443         for my $target (keys $bridges->%*) {
 444             for my $nic (keys $bridges->{$target}->%*) {
 445                 $self->log('info', "mapped: $nic from $bridges->{$target}->{$nic} to $target");
 446             }
 447         }
 448         my $conf_str = PVE::LXC::Config::write_pct_config("remote", $remote_conf);
 449
 450         # TODO expose in PVE::Firewall?
 451         my $vm_fw_conf_path = "/etc/pve/firewall/$vmid.fw";
 452         my $fw_conf_str;
 453         $fw_conf_str = PVE::Tools::file_get_contents($vm_fw_conf_path)
 454             if -e $vm_fw_conf_path;
 455         my $params = {
 456             conf => $conf_str,
 457             'firewall-config' => $fw_conf_str,
 458         };
 459
 460         PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'config', $params);
 461     } else {
 462         # transfer replication state before moving config
 463         $self->transfer_replication_state() if $rep_volumes;
 464         PVE::LXC::Config->update_volume_ids($conf, $self->{volume_map});
 465         PVE::LXC::Config->write_config($vmid, $conf);
 466         PVE::LXC::Config->move_config_to_node($vmid, $self->{node});
 467         $self->switch_replication_job_target() if $rep_volumes;
 468     }
 469     $self->{conf_migrated} = 1;
 470 }
 471
 472 sub phase1_cleanup {
 473     my ($self, $vmid, $err) = @_;
 474
 475     $self->log('info', "aborting phase 1 - cleanup resources");
 476
 477     if ($self->{volumes}) {
 478         foreach my $volid (@{$self->{volumes}}) {
 479             if (my $mapped_volume = $self->{volume_map}->{$volid}) {
 480                 $volid = $mapped_volume;
 481             }
 482             $self->log('err', "found stale volume copy '$volid' on node '$self->{node}'");
 483             # fixme: try to remove ?
 484         }
 485     }
 486
 487     if ($self->{opts}->{remote}) {
 488         # cleans up remote volumes
 489         PVE::Tunnel::finish_tunnel($self->{tunnel}, 1);
 490         delete $self->{tunnel};
 491     }
 492 }
 493
 494 sub phase3 {
 495     my ($self, $vmid) = @_;
 496
 497     my $volids = $self->{volumes};
 498
 499     # handled below in final_cleanup
 500     return if $self->{opts}->{remote};
 501
 502     # destroy local copies
 503     foreach my $volid (@$volids) {
 504         eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
 505         if (my $err = $@) {
 506             $self->log('err', "removing local copy of '$volid' failed - $err");
 507             $self->{errors} = 1;
 508             last if $err =~ /^interrupted by signal$/;
 509         }
 510     }
 511 }
 512
 513 sub final_cleanup {
 514     my ($self, $vmid) = @_;
 515
 516     $self->log('info', "start final cleanup");
 517
 518     if (!$self->{conf_migrated}) {
 519         eval { PVE::LXC::Config->remove_lock($vmid, 'migrate'); };
 520         if (my $err = $@) {
 521             $self->log('err', $err);
 522         }
 523         # in restart mode, we start the container on the source node on migration error
 524         if ($self->{opts}->{restart} && $self->{was_running}) {
 525             $self->log('info', "start container on source node");
 526             my $skiplock = 1;
 527             PVE::LXC::vm_start($vmid, $self->{vmconf}, $skiplock);
 528         }
 529     } elsif ($self->{opts}->{remote}) {
 530         eval { PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'unlock') };
 531         $self->log('err', "Failed to clear migrate lock - $@\n") if $@;
 532
 533         if ($self->{opts}->{restart} && $self->{was_running}) {
 534             $self->log('info', "start container on target node");
 535             PVE::Tunnel::write_tunnel($self->{tunnel}, 60, 'start');
 536         }
 537         if ($self->{opts}->{delete}) {
 538             PVE::LXC::destroy_lxc_container(
 539                 PVE::Storage::config(),
 540                 $vmid,
 541                 PVE::LXC::Config->load_config($vmid),
 542                 undef,
 543                 0,
 544             );
 545         }
 546         PVE::Tunnel::finish_tunnel($self->{tunnel});
 547     } else {
 548         my $cmd = [ @{$self->{rem_ssh}}, 'pct', 'unlock', $vmid ];
 549         $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
 550
 551         # in restart mode, we start the container on the target node after migration
 552         if ($self->{opts}->{restart} && $self->{was_running}) {
 553             $self->log('info', "start container on target node");
 554             my $cmd = [ @{$self->{rem_ssh}}, 'pct', 'start', $vmid];
 555             $self->cmd($cmd);
 556         }
 557     }
 558 }
 559
 560 sub map_bridges {
 561     my ($conf, $map, $scan_only) = @_;
 562
 563     my $bridges = {};
 564
 565     foreach my $opt (keys %$conf) {
 566         next if $opt !~ m/^net\d+$/;
 567
 568         next if !$conf->{$opt};
 569         my $d = PVE::LXC::Config->parse_lxc_network($conf->{$opt});
 570         next if !$d || !$d->{bridge};
 571
 572         my $target_bridge = PVE::JSONSchema::map_id($map, $d->{bridge});
 573         $bridges->{$target_bridge}->{$opt} = $d->{bridge};
 574
 575         next if $scan_only;
 576
 577         $d->{bridge} = $target_bridge;
 578         $conf->{$opt} = PVE::LXC::Config->print_lxc_network($d);
 579     }
 580
 581     return $bridges;
 582 }
 583
 584 1;