]> git.proxmox.com Git - pve-container.git/blame - src/PVE/LXC/Migrate.pm
buildsys: change upload dist to bullseye
[pve-container.git] / src / PVE / LXC / Migrate.pm
CommitLineData
6f42807e
DM
1package PVE::LXC::Migrate;
2
3use strict;
4use warnings;
5use PVE::AbstractMigrate;
6use File::Basename;
7use File::Copy; # fixme: remove
8use PVE::Tools;
9use PVE::INotify;
10use PVE::Cluster;
11use PVE::Storage;
8d66edee 12use PVE::LXC::Config;
6f42807e 13use PVE::LXC;
3efa5e3d
DM
14use PVE::ReplicationConfig;
15use PVE::ReplicationState;
16use PVE::Replication;
6f42807e
DM
17
18use base qw(PVE::AbstractMigrate);
19
20sub lock_vm {
21 my ($self, $vmid, $code, @param) = @_;
22
67afe46e 23 return PVE::LXC::Config->lock_config($vmid, $code, @param);
6f42807e
DM
24}
25
26sub prepare {
27 my ($self, $vmid) = @_;
28
29 my $online = $self->{opts}->{online};
a7cedb73 30 my $restart= $self->{opts}->{restart};
6f42807e
DM
31
32 $self->{storecfg} = PVE::Storage::config();
33
1cd1fa12 34 # test if CT exists
67afe46e 35 my $conf = $self->{vmconf} = PVE::LXC::Config->load_config($vmid);
6f42807e 36
67afe46e 37 PVE::LXC::Config->check_lock($conf);
6f42807e
DM
38
39 my $running = 0;
40 if (PVE::LXC::check_running($vmid)) {
a7cedb73
DC
41 die "lxc live migration is currently not implemented\n" if $online;
42 die "running container can only be migrated in restart mode" if !$restart;
6f42807e
DM
43 $running = 1;
44 }
a7cedb73 45 $self->{was_running} = $running;
6f42807e 46
9746c095
FG
47 my $force = $self->{opts}->{force} // 0;
48
015740e6 49 PVE::LXC::Config->foreach_volume($conf, sub {
6f42807e
DM
50 my ($ms, $mountpoint) = @_;
51
52 my $volid = $mountpoint->{volume};
552e168f 53 my $type = $mountpoint->{type};
9746c095 54
552e168f
FG
55 # skip dev/bind mps when forced / shared
56 if ($type ne 'volume') {
57 if ($force) {
58 warn "-force is deprecated, please use the 'shared' property on individual non-volume mount points instead!\n";
59 return;
60 }
61 if ($mountpoint->{shared}) {
62 return;
63 } else {
64 die "cannot migrate local $type mount point '$ms'\n";
65 }
9746c095 66 }
552e168f 67
6f42807e 68 my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1) if $volid;
235dbdf3 69 die "can't determine assigned storage for mount point '$ms'\n" if !$storage;
6f42807e
DM
70
71 # check if storage is available on both nodes
72 my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $storage);
73 PVE::Storage::storage_check_node($self->{storecfg}, $storage, $self->{node});
74
20ab40f3
FG
75
76 if ($scfg->{shared}) {
77 # PVE::Storage::activate_storage checks this for non-shared storages
78 my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
79 warn "Used shared storage '$storage' is not online on source node!\n"
80 if !$plugin->check_connection($storage, $scfg);
81 } else {
a7cedb73 82 # unless in restart mode because we shut the container down
235dbdf3 83 die "unable to migrate local mount point '$volid' while CT is running"
a7cedb73 84 if $running && !$restart;
20ab40f3 85 }
6f42807e
DM
86
87 });
88
6f42807e
DM
89 # todo: test if VM uses local resources
90
91 # test ssh connection
92 my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
93 eval { $self->cmd_quiet($cmd); };
94 die "Can't connect to destination address using public key\n" if $@;
95
a7cedb73
DC
96 # in restart mode, we shutdown the container before migrating
97 if ($restart && $running) {
98 my $timeout = $self->{opts}->{timeout} // 180;
99
100 $self->log('info', "shutdown CT $vmid\n");
101
b1bad293 102 PVE::LXC::vm_stop($vmid, 0, $timeout);
a7cedb73
DC
103
104 $running = 0;
105 }
106
6f42807e
DM
107 return $running;
108}
109
110sub phase1 {
111 my ($self, $vmid) = @_;
112
113 $self->log('info', "starting migration of CT $self->{vmid} to node '$self->{node}' ($self->{nodeip})");
114
115 my $conf = $self->{vmconf};
116 $conf->{lock} = 'migrate';
67afe46e 117 PVE::LXC::Config->write_config($vmid, $conf);
6f42807e
DM
118
119 if ($self->{running}) {
120 $self->log('info', "container is running - using online migration");
121 }
122
3c5dabe1 123 $self->{volumes} = []; # list of already migrated volumes
f503617b 124 my $volhash = {}; # 'config', 'snapshot' or 'storage' for local volumes
33deb76c
FG
125 my $volhash_errors = {};
126 my $abort = 0;
127
128 my $log_error = sub {
129 my ($msg, $volid) = @_;
130
131 $volhash_errors->{$volid} = $msg if !defined($volhash_errors->{$volid});
132 $abort = 1;
133 };
6f42807e 134
3c5dabe1
FG
135 my $test_volid = sub {
136 my ($volid, $snapname) = @_;
6f42807e 137
3c5dabe1
FG
138 return if !$volid;
139
140 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
141
142 # check if storage is available on both nodes
143 my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid);
144 PVE::Storage::storage_check_node($self->{storecfg}, $sid, $self->{node});
145
33deb76c
FG
146 if ($scfg->{shared}) {
147 $self->log('info', "volume '$volid' is on shared storage '$sid'")
148 if !$snapname;
149 return;
150 }
3c5dabe1 151
0aa2d2a2
WB
152 $volhash->{$volid}->{ref} = defined($snapname) ? 'snapshot' : 'config';
153 $volhash->{$volid}->{snapshots} = defined($snapname);
f503617b 154
3c5dabe1
FG
155 my ($path, $owner) = PVE::Storage::path($self->{storecfg}, $volid);
156
33deb76c 157 die "owned by other guest (owner = $owner)\n"
3c5dabe1 158 if !$owner || ($owner != $self->{vmid});
9746c095 159
3c5dabe1
FG
160 if (defined($snapname)) {
161 # we cannot migrate shapshots on local storage
162 # exceptions: 'zfspool'
163 if (($scfg->{type} eq 'zfspool')) {
3c5dabe1
FG
164 return;
165 }
33deb76c 166 die "non-migratable snapshot exists\n";
3c5dabe1
FG
167 }
168 };
169
170 my $test_mp = sub {
171 my ($ms, $mountpoint, $snapname) = @_;
172
173 my $volid = $mountpoint->{volume};
9746c095
FG
174 # already checked in prepare
175 if ($mountpoint->{type} ne 'volume') {
552e168f 176 $self->log('info', "ignoring shared '$mountpoint->{type}' mount point '$ms' ('$volid')")
3c5dabe1 177 if !$snapname;
9746c095
FG
178 return;
179 }
180
33deb76c
FG
181 eval {
182 &$test_volid($volid, $snapname);
183 };
6f42807e 184
33deb76c 185 &$log_error($@, $volid) if $@;
3c5dabe1
FG
186 };
187
1c5f78ef 188 # first unused / lost volumes owned by this container
3c5dabe1
FG
189 my @sids = PVE::Storage::storage_ids($self->{storecfg});
190 foreach my $storeid (@sids) {
191 my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
192 next if $scfg->{shared};
193 next if !PVE::Storage::storage_check_enabled($self->{storecfg}, $storeid, undef, 1);
194
195 # get list from PVE::Storage (for unused volumes)
196 my $dl = PVE::Storage::vdisk_list($self->{storecfg}, $storeid, $vmid);
197
198 next if @{$dl->{$storeid}} == 0;
199
200 # check if storage is available on target node
201 PVE::Storage::storage_check_node($self->{storecfg}, $storeid, $self->{node});
202
203 PVE::Storage::foreach_volid($dl, sub {
204 my ($volid, $sid, $volname) = @_;
205
0aa2d2a2 206 $volhash->{$volid}->{ref} = 'storage';
3c5dabe1
FG
207 });
208 }
209
1c5f78ef
FG
210 # then all volumes referenced in snapshots
211 foreach my $snapname (keys %{$conf->{snapshots}}) {
212 &$test_volid($conf->{snapshots}->{$snapname}->{'vmstate'}, 0, undef)
213 if defined($conf->{snapshots}->{$snapname}->{'vmstate'});
015740e6 214 PVE::LXC::Config->foreach_volume($conf->{snapshots}->{$snapname}, $test_mp, $snapname);
1c5f78ef
FG
215 }
216
217 # finally all currently used volumes
015740e6 218 PVE::LXC::Config->foreach_volume($conf, $test_mp);
1c5f78ef
FG
219
220
3c5dabe1
FG
221 # additional checks for local storage
222 foreach my $volid (keys %$volhash) {
33deb76c
FG
223 eval {
224 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
225 my $scfg = PVE::Storage::storage_config($self->{storecfg}, $sid);
3c5dabe1 226
33deb76c
FG
227 my $migratable = ($scfg->{type} eq 'dir') || ($scfg->{type} eq 'zfspool') ||
228 ($scfg->{type} eq 'lvmthin') || ($scfg->{type} eq 'lvm');
3c5dabe1 229
33deb76c
FG
230 die "storage type '$scfg->{type}' not supported\n"
231 if !$migratable;
3c5dabe1 232
33deb76c
FG
233 # image is a linked clone on local storage, se we can't migrate.
234 if (my $basename = (PVE::Storage::parse_volname($self->{storecfg}, $volid))[3]) {
235 die "clone of '$basename'";
236 }
237 };
238 &$log_error($@, $volid) if $@;
3c5dabe1
FG
239 }
240
f503617b 241 foreach my $volid (sort keys %$volhash) {
0aa2d2a2
WB
242 my $ref = $volhash->{$volid}->{ref};
243 if ($ref eq 'storage') {
f503617b 244 $self->log('info', "found local volume '$volid' (via storage)\n");
0aa2d2a2 245 } elsif ($ref eq 'config') {
f503617b 246 $self->log('info', "found local volume '$volid' (in current VM config)\n");
0aa2d2a2 247 } elsif ($ref eq 'snapshot') {
f503617b
FG
248 $self->log('info', "found local volume '$volid' (referenced by snapshot(s))\n");
249 } else {
250 $self->log('info', "found local volume '$volid'\n");
251 }
252 }
253
33deb76c
FG
254 foreach my $volid (sort keys %$volhash_errors) {
255 $self->log('warn', "can't migrate local volume '$volid': $volhash_errors->{$volid}");
256 }
257
258 if ($abort) {
259 die "can't migrate CT - check log\n";
260 }
261
3efa5e3d
DM
262 my $rep_volumes;
263
264 my $rep_cfg = PVE::ReplicationConfig->new();
265
266 if (my $jobcfg = $rep_cfg->find_local_replication_job($vmid, $self->{node})) {
267 die "can't live migrate VM with replicated volumes\n" if $self->{running};
268 my $start_time = time();
269 my $logfunc = sub { my ($msg) = @_; $self->log('info', $msg); };
270 $rep_volumes = PVE::Replication::run_replication(
271 'PVE::LXC::Config', $jobcfg, $start_time, $start_time, $logfunc);
272 }
273
1159329a 274 my $opts = $self->{opts};
3c5dabe1 275 foreach my $volid (keys %$volhash) {
3efa5e3d 276 next if $rep_volumes->{$volid};
3c5dabe1
FG
277 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
278 push @{$self->{volumes}}, $volid;
5aa2a703 279 my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', [$sid], $opts->{bwlimit});
1159329a
SI
280 # JSONSchema and get_bandwidth_limit use kbps - storage_migrate bps
281 $bwlimit = $bwlimit * 1024 if defined($bwlimit);
282
1d26bb86 283 my $storage_migrate_opts = {
a799df7a 284 'ratelimit_bps' => $bwlimit,
1d26bb86
FE
285 'insecure' => $opts->{migration_type} eq 'insecure',
286 'with_snapshots' => $volhash->{$volid}->{snapshots},
287 };
288
d79b051a
FE
289 my $logfunc = sub { $self->log('info', $_[0]); };
290 eval {
291 PVE::Storage::storage_migrate($self->{storecfg}, $volid, $self->{ssh_info},
292 $sid, $storage_migrate_opts, $logfunc);
293 };
294 if (my $err = $@) {
295 die "storage migration for '$volid' to storage '$sid' failed - $err\n";
296 }
1e5f5da7
FE
297
298 eval { PVE::Storage::deactivate_volumes($self->{storecfg}, [$volid]); };
299 if (my $err = $@) {
300 $self->log('warn', $err);
301 }
3c5dabe1 302 }
6f42807e 303
6f42807e
DM
304 if ($self->{running}) {
305 die "implement me";
306 }
307
308 # make sure everything on (shared) storage is unmounted
309 # Note: we must be 100% sure, else we get data corruption because
310 # non-shared file system could be mounted twice (on shared storage)
311
312 PVE::LXC::umount_all($vmid, $self->{storecfg}, $conf);
313
c9bc5018 314 #to be sure there are no active volumes
d250604f 315 my $vollist = PVE::LXC::Config->get_vm_volumes($conf);
c9bc5018
WL
316 PVE::Storage::deactivate_volumes($self->{storecfg}, $vollist);
317
86df609a 318 # transfer replication state before moving config
522d3569 319 $self->transfer_replication_state() if $rep_volumes;
86df609a 320 PVE::LXC::Config->move_config_to_node($vmid, $self->{node});
1d1c1b4f 321 $self->{conf_migrated} = 1;
522d3569 322 $self->switch_replication_job_target() if $rep_volumes;
6f42807e
DM
323}
324
325sub phase1_cleanup {
326 my ($self, $vmid, $err) = @_;
327
328 $self->log('info', "aborting phase 1 - cleanup resources");
329
330 if ($self->{volumes}) {
331 foreach my $volid (@{$self->{volumes}}) {
332 $self->log('err', "found stale volume copy '$volid' on node '$self->{node}'");
333 # fixme: try to remove ?
334 }
335 }
336}
337
338sub phase3 {
339 my ($self, $vmid) = @_;
340
341 my $volids = $self->{volumes};
342
343 # destroy local copies
344 foreach my $volid (@$volids) {
8fdd1fc8
DM
345 eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
346 if (my $err = $@) {
347 $self->log('err', "removing local copy of '$volid' failed - $err");
348 $self->{errors} = 1;
349 last if $err =~ /^interrupted by signal$/;
6f42807e
DM
350 }
351 }
352}
353
354sub final_cleanup {
355 my ($self, $vmid) = @_;
356
357 $self->log('info', "start final cleanup");
358
1d1c1b4f 359 if (!$self->{conf_migrated}) {
dd3bcfb2 360 eval { PVE::LXC::Config->remove_lock($vmid, 'migrate'); };
1d1c1b4f
WL
361 if (my $err = $@) {
362 $self->log('err', $err);
363 }
6725e93c
AA
364 # in restart mode, we start the container on the source node
365 # on migration error
366 if ($self->{opts}->{restart} && $self->{was_running}) {
367 $self->log('info', "start container on source node");
368 my $skiplock = 1;
369 PVE::LXC::vm_start($vmid, $self->{vmconf}, $skiplock);
370 }
1d1c1b4f
WL
371 } else {
372 my $cmd = [ @{$self->{rem_ssh}}, 'pct', 'unlock', $vmid ];
ee8d9207
FG
373 $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
374
375 # in restart mode, we start the container on the target node
376 # after migration
377 if ($self->{opts}->{restart} && $self->{was_running}) {
378 $self->log('info', "start container on target node");
379 my $cmd = [ @{$self->{rem_ssh}}, 'pct', 'start', $vmid];
380 $self->cmd($cmd);
381 }
6f42807e 382 }
a7cedb73 383
6f42807e
DM
384}
385
3861;