1 package PVE
::VZDump
::QemuServer
;
12 use PVE
::Cluster
qw(cfs_read_file);
17 use PVE
::Storage
::Plugin
;
18 use PVE
::Storage
::PBSPlugin
;
25 use PVE
::QemuServer
::Machine
;
26 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
28 use base qw
(PVE
::VZDump
::Plugin
);
31 my ($class, $vzdump) = @_;
33 PVE
::VZDump
::check_bin
('qm');
35 my $self = bless { vzdump
=> $vzdump }, $class;
37 $self->{vmlist
} = PVE
::QemuServer
::vzlist
();
38 $self->{storecfg
} = PVE
::Storage
::config
();
49 return [ keys %{$self->{vmlist
}} ];
53 my ($self, $task, $vmid, $mode) = @_;
57 my $conf = $self->{vmlist
}->{$vmid} = PVE
::QemuConfig-
>load_config($vmid);
59 $self->loginfo("VM Name: $conf->{name}")
60 if defined($conf->{name
});
62 $self->{vm_was_running
} = 1;
63 if (!PVE
::QemuServer
::check_running
($vmid)) {
64 $self->{vm_was_running
} = 0;
67 $task->{hostname
} = $conf->{name
};
69 my $hostname = PVE
::INotify
::nodename
();
73 my $backup_volumes = PVE
::QemuConfig-
>get_backup_volumes($conf);
75 foreach my $volume (@{$backup_volumes}) {
76 my $name = $volume->{key
};
77 my $volume_config = $volume->{volume_config
};
78 my $volid = $volume_config->{file
};
80 if (!$volume->{included
}) {
81 $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})");
83 } elsif ($self->{vm_was_running
} && $volume_config->{iothread
}) {
84 if (!PVE
::QemuServer
::Machine
::runs_at_least_qemu_version
($vmid, 4, 0, 1)) {
85 die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " .
86 "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
89 my $log = "include disk '$name' '$volid'";
90 if (defined(my $size = $volume_config->{size
})) {
91 my $readable_size = PVE
::JSONSchema
::format_size
($size);
92 $log .= " $readable_size";
97 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
98 push @$vollist, $volid if $storeid;
99 $drivehash->{$name} = $volume->{volume_config
};
102 PVE
::Storage
::activate_volumes
($self->{storecfg
}, $vollist);
104 foreach my $ds (sort keys %$drivehash) {
105 my $drive = $drivehash->{$ds};
107 my $volid = $drive->{file
};
108 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
112 $path = PVE
::Storage
::path
($self->{storecfg
}, $volid);
116 my ($size, $format) = eval { PVE
::Storage
::volume_size_info
($self->{storecfg
}, $volid, 5) };
117 die "no such volume '$volid'\n" if $@;
125 qmdevice
=> "drive-$ds",
129 $diskinfo->{type
} = 'block';
131 $diskinfo->{type
} = 'file';
134 push @{$task->{disks
}}, $diskinfo;
139 my ($self, $vmid) = @_;
141 my $running = PVE
::QemuServer
::check_running
($vmid) ?
1 : 0;
143 return wantarray ?
($running, $running ?
'running' : 'stopped') : $running;
147 my ($self, $vmid) = @_;
149 PVE
::QemuConfig-
>set_lock($vmid, 'backup');
153 my ($self, $vmid) = @_;
155 PVE
::QemuConfig-
>remove_lock($vmid, 'backup');
159 my ($self, $task, $vmid) = @_;
161 my $opts = $self->{vzdump
}->{opts
};
163 my $wait = $opts->{stopwait
} * 60;
164 # send shutdown and wait
165 $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
169 my ($self, $task, $vmid) = @_;
171 $self->cmd ("qm start $vmid --skiplock");
175 my ($self, $task, $vmid) = @_;
177 $self->cmd ("qm suspend $vmid --skiplock");
181 my ($self, $task, $vmid) = @_;
183 $self->cmd ("qm resume $vmid --skiplock");
187 my ($self, $task, $vmid) = @_;
189 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
191 my $outfile = "$task->{tmpdir}/qemu-server.conf";
192 my $firewall_src = "/etc/pve/firewall/$vmid.fw";
193 my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
195 my $outfd = IO
::File-
>new (">$outfile") ||
196 die "unable to open '$outfile'";
197 my $conffd = IO
::File-
>new ($conffile, 'r') ||
198 die "unable open '$conffile'";
202 while (defined (my $line = <$conffd>)) {
203 next if $line =~ m/^\#vzdump\#/; # just to be sure
204 next if $line =~ m/^\#qmdump\#/; # just to be sure
205 if ($line =~ m/^\[(.*)\]\s*$/) {
206 if ($1 =~ m/PENDING/i) {
212 next if $found_snapshot || $found_pending; # skip all snapshots and pending changes config data
214 if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
215 $self->loginfo("skip unused drive '$1' (not included into backup)");
218 next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
223 foreach my $di (@{$task->{disks
}}) {
224 if ($di->{type
} eq 'block' || $di->{type
} eq 'file') {
225 my $storeid = $di->{storeid
} || '';
226 my $format = $di->{format
} || '';
227 print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
229 die "internal error";
233 if ($found_snapshot) {
234 $self->loginfo("snapshots found (not included into backup)");
236 if ($found_pending) {
237 $self->loginfo("pending configuration changes found (not included into backup)");
240 PVE
::Tools
::file_copy
($firewall_src, $firewall_dest) if -f
$firewall_src;
244 my ($self, $task, $vmid, $filename, $comp) = @_;
246 my $opts = $self->{vzdump
}->{opts
};
247 my $scfg = $opts->{scfg
};
249 if ($self->{vzdump
}->{opts
}->{pbs
}) {
250 $self->archive_pbs($task, $vmid);
252 $self->archive_vma($task, $vmid, $filename, $comp);
256 # number, [precision=1]
258 return sprintf( "%." . ( $_[1] || 1 ) . "f", $_[0] );
261 my ($bytes, $precission) = @_;
263 return $num2str->($bytes, $precission) . ' B' if $bytes < 1024;
264 my $kb = $bytes/1024;
266 return $num2str->($kb, $precission) . " KiB" if $kb < 1024;
269 return $num2str->($mb, $precission) . " MiB" if $mb < 1024;
272 return $num2str->($gb, $precission) . " GiB" if $gb < 1024;
275 return $num2str->($tb, $precission) . " TiB";
278 my $bitmap_action_to_human = sub {
279 my ($self, $info) = @_;
281 my $action = $info->{action
};
283 if ($action eq "not-used") {
284 return "disabled (no support)" if $self->{vm_was_running
};
285 return "disabled (VM not running)";
286 } elsif ($action eq "not-used-removed") {
287 return "disabled (old bitmap cleared)";
288 } elsif ($action eq "new") {
289 return "created new";
290 } elsif ($action eq "used") {
291 if ($info->{dirty
} == 0) {
292 return "OK (drive clean)";
294 my $size = bytes_to_human
($info->{size
});
295 my $dirty = bytes_to_human
($info->{dirty
});
296 return "OK ($dirty of $size dirty)";
298 } elsif ($action eq "invalid") {
299 return "existing bitmap was invalid and has been cleared";
305 my $query_backup_status_loop = sub {
306 my ($self, $vmid, $job_uuid, $pbs_features) = @_;
308 my $starttime = time ();
309 my $last_time = $starttime;
310 my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0);
311 my ($transferred, $reused);
314 my ($mb, $delta) = @_;
315 return "0 B/s" if $mb <= 0;
316 my $bw = int(($mb / $delta));
317 return bytes_to_human
($bw) . "/s";
322 my $has_query_bitmap = 0;
323 if (defined($pbs_features) && $pbs_features->{'query-bitmap-info'}) {
324 $has_query_bitmap = 1;
326 my $bitmap_info = mon_cmd
($vmid, 'query-pbs-bitmap-info');
327 foreach my $info (sort { $a->{drive
} cmp $b->{drive
} } @$bitmap_info) {
328 my $text = $bitmap_action_to_human->($self, $info);
329 my $drive = $info->{drive
};
330 $drive =~ s/^drive-//; # for consistency
331 $self->loginfo("$drive: dirty-bitmap status: $text");
332 $target += $info->{dirty
};
333 $total += $info->{size
};
334 $last_reused += $info->{size
} - $info->{dirty
};
336 if ($target < $total) {
337 my $total_h = bytes_to_human
($total);
338 my $target_h = bytes_to_human
($target);
339 $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
344 my $status = mon_cmd
($vmid, 'query-backup');
346 my $total = $status->{total
} || 0;
347 $target = $total if !$has_query_bitmap;
348 $transferred = $status->{transferred
} || 0;
349 $reused = $status->{reused
};
350 my $percent = $target ?
int(($transferred * 100)/$target) : 100;
351 my $zero = $status->{'zero-bytes'} || 0;
353 die "got unexpected uuid\n" if !$status->{uuid
} || ($status->{uuid
} ne $job_uuid);
356 my $duration = $ctime - $starttime;
358 my $rbytes = $transferred - $last_transferred;
361 # reused includes zero bytes for PBS
362 $wbytes = $rbytes - ($reused - $last_reused);
364 $wbytes = $rbytes - ($zero - $last_zero);
367 my $timediff = ($ctime - $last_time) || 1; # fixme
368 my $mbps_read = $get_mbps->($rbytes, $timediff);
369 my $mbps_write = $get_mbps->($wbytes, $timediff);
370 my $target_h = bytes_to_human
($target);
371 my $transferred_h = bytes_to_human
($transferred);
373 my $statusline = "status: $percent% ($transferred_h of $target_h), duration $duration"
374 .", read: $mbps_read, write: $mbps_write";
376 my $res = $status->{status
} || 'unknown';
377 if ($res ne 'active') {
378 $self->loginfo($statusline);
379 if ($res ne 'done') {
380 die (($status->{errmsg
} || "unknown error") . "\n") if $res eq 'error';
381 die "got unexpected status '$res'\n";
383 $last_target = $target if $target;
384 $last_total = $total if $total;
385 $last_zero = $zero if $zero;
386 $last_transferred = $transferred if $transferred;
389 if ($percent != $last_percent && ($timediff > 2)) {
390 $self->loginfo($statusline);
391 $last_percent = $percent;
392 $last_target = $target if $target;
393 $last_total = $total if $total;
394 $last_zero = $zero if $zero;
395 $last_transferred = $transferred if $transferred;
397 $last_reused = $reused;
402 my $duration = time() - $starttime;
405 my $zero_per = $last_target ?
int(($last_zero * 100)/$last_target) : 0;
406 my $zero_h = bytes_to_human
($last_zero, 2);
407 $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
410 my $transferred_h = bytes_to_human
($transferred, 2);
412 my $reused_h = bytes_to_human
($reused, 2);
413 my $reuse_per = int($reused * 100 / $last_total);
414 $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
417 my $mbps = $get_mbps->($transferred, $duration);
418 $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
420 $self->loginfo("transferred $transferred_h in <1 seconds");
425 total
=> $last_total,
431 my ($self, $task, $vmid) = @_;
433 my $conffile = "$task->{tmpdir}/qemu-server.conf";
434 my $firewall = "$task->{tmpdir}/qemu-server.fw";
436 my $opts = $self->{vzdump
}->{opts
};
437 my $scfg = $opts->{scfg
};
439 my $starttime = time();
441 my $server = $scfg->{server
};
442 my $datastore = $scfg->{datastore
};
443 my $username = $scfg->{username
} // 'root@pam';
444 my $fingerprint = $scfg->{fingerprint
};
446 my $repo = "$username\@$server:$datastore";
447 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $opts->{storage
});
448 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($scfg, $opts->{storage
});
450 my $diskcount = scalar(@{$task->{disks
}});
451 if (PVE
::QemuConfig-
>is_template($self->{vmlist
}->{$vmid}) || !$diskcount) {
453 # FIXME: accumulate disk sizes to use for backup job (email) log
454 foreach my $di (@{$task->{disks
}}) {
455 if ($di->{type
} eq 'block' || $di->{type
} eq 'file') {
456 push @pathlist, "$di->{qmdevice}.img:$di->{path}";
458 die "implement me (type $di->{type})";
463 $self->loginfo("backup contains no disks");
466 local $ENV{PBS_PASSWORD
} = $password;
467 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
469 '/usr/bin/proxmox-backup-client',
471 '--repository', $repo,
472 '--backup-type', 'vm',
473 '--backup-id', "$vmid",
474 '--backup-time', $task->{backup_time
},
477 push @$cmd, "qemu-server.conf:$conffile";
478 push @$cmd, "fw.conf:$firewall" if -e
$firewall;
479 push @$cmd, @pathlist if scalar(@pathlist);
481 $self->loginfo("starting template backup");
482 $self->loginfo(join(' ', @$cmd));
489 # get list early so we die on unkown drive types before doing anything
490 my $devlist = _get_task_devlist
($task);
492 $self->enforce_vm_running_for_backup($vmid);
496 $SIG{INT
} = $SIG{TERM
} = $SIG{QUIT
} = $SIG{HUP
} = $SIG{PIPE
} = sub {
497 die "interrupted by signal\n";
500 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
501 if (!$qemu_support) {
502 die "PBS backups are not supported by the running QEMU version. Please make "
503 . "sure you've installed the latest version and the VM has been restarted.\n";
506 my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
510 'backup-file' => $repo,
511 'backup-id' => "$vmid",
512 'backup-time' => $task->{backup_time
},
513 password
=> $password,
515 'config-file' => $conffile,
517 $params->{fingerprint
} = $fingerprint if defined($fingerprint);
518 $params->{'firewall-file'} = $firewall if -e
$firewall;
520 $self->loginfo("enabling encryption");
521 $params->{keyfile
} = $keyfile;
522 $params->{encrypt
} = JSON
::true
;
524 $params->{encrypt
} = JSON
::false
;
527 $params->{'use-dirty-bitmap'} = JSON
::true
528 if $qemu_support->{'pbs-dirty-bitmap'} && $self->{vm_was_running
};
530 $params->{timeout
} = 60; # give some time to connect to the backup server
532 my $res = eval { mon_cmd
($vmid, "backup", %$params) };
534 $backup_job_uuid = $res->{UUID
} if $res;
537 $self->qga_fs_thaw($vmid);
540 die $qmperr if $qmperr;
541 die "got no uuid for backup task\n" if !defined($backup_job_uuid);
543 $self->loginfo("started backup task '$backup_job_uuid'");
545 $self->resume_vm_after_job_start($task, $vmid);
547 my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support);
548 $task->{size
} = $stat->{total
};
553 $self->mon_backup_cancel($vmid) if defined($backup_job_uuid);
555 $self->restore_vm_power_state($vmid);
560 my $fork_compressor_pipe = sub {
561 my ($self, $comp, $outfileno) = @_;
563 my @pipefd = POSIX
::pipe();
565 die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0;
568 POSIX
::close($pipefd[1]);
570 my $fd = fileno(STDIN
);
572 POSIX
::close(0) if $fd != 0;
573 die "unable to redirect STDIN - $!"
574 if !open(STDIN
, "<&", $pipefd[0]);
577 $fd = fileno(STDOUT
);
579 POSIX
::close (1) if $fd != 1;
581 die "unable to redirect STDOUT - $!"
582 if !open(STDOUT
, ">&", $outfileno);
585 die "fork compressor '$comp' failed\n";
594 POSIX
::close($pipefd[0]);
595 $outfileno = $pipefd[1];
598 return ($cpid, $outfileno);
602 my ($self, $task, $vmid, $filename, $comp) = @_;
604 my $conffile = "$task->{tmpdir}/qemu-server.conf";
605 my $firewall = "$task->{tmpdir}/qemu-server.fw";
607 my $opts = $self->{vzdump
}->{opts
};
609 my $starttime = time();
612 if ($opts->{bwlimit
}) {
613 $speed = $opts->{bwlimit
}*1024;
616 my $diskcount = scalar(@{$task->{disks
}});
617 if (PVE
::QemuConfig-
>is_template($self->{vmlist
}->{$vmid}) || !$diskcount) {
619 foreach my $di (@{$task->{disks
}}) {
620 if ($di->{type
} eq 'block' || $di->{type
} eq 'file') {
621 push @pathlist, "$di->{qmdevice}=$di->{path}";
628 $self->loginfo("backup contains no disks");
633 $outcmd = "exec:$comp";
635 $outcmd = "exec:cat";
638 $outcmd .= " > $filename" if !$opts->{stdout
};
640 my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
641 push @$cmd, '-c', $firewall if -e
$firewall;
642 push @$cmd, $outcmd, @pathlist;
644 $self->loginfo("starting template backup");
645 $self->loginfo(join(' ', @$cmd));
647 if ($opts->{stdout
}) {
648 $self->cmd($cmd, output
=> ">&" . fileno($opts->{stdout
}));
656 my $devlist = _get_task_devlist
($task);
658 $self->enforce_vm_running_for_backup($vmid);
664 $SIG{INT
} = $SIG{TERM
} = $SIG{QUIT
} = $SIG{HUP
} = $SIG{PIPE
} = sub {
665 die "interrupted by signal\n";
669 if ($opts->{stdout
}) {
670 $outfh = $opts->{stdout
};
672 $outfh = IO
::File-
>new($filename, "w") ||
673 die "unable to open file '$filename' - $!\n";
675 my $outfileno = fileno($outfh);
678 ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno);
681 my $qmpclient = PVE
::QMPClient-
>new();
682 my $backup_cb = sub {
683 my ($vmid, $resp) = @_;
684 $backup_job_uuid = $resp->{return}->{UUID
};
686 my $add_fd_cb = sub {
687 my ($vmid, $resp) = @_;
690 'backup-file' => "/dev/fdname/backup",
692 'config-file' => $conffile,
695 $params->{'firewall-file'} = $firewall if -e
$firewall;
697 $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
700 $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd', fd
=> $outfileno, fdname
=> "backup");
702 my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
704 eval { $qmpclient->queue_execute(30) };
708 $self->qga_fs_thaw($vmid);
711 die $qmperr if $qmperr;
712 die $qmpclient->{errors
}->{$vmid} if $qmpclient->{errors
}->{$vmid};
715 POSIX
::close($outfileno) == 0 ||
716 die "close output file handle failed\n";
719 die "got no uuid for backup task\n" if !defined($backup_job_uuid);
721 $self->loginfo("started backup task '$backup_job_uuid'");
723 $self->resume_vm_after_job_start($task, $vmid);
725 $query_backup_status_loop->($self, $vmid, $backup_job_uuid);
730 $self->mon_backup_cancel($vmid) if defined($backup_job_uuid);
733 $self->restore_vm_power_state($vmid);
743 if ($cpid && (waitpid($cpid, 0) > 0)) {
746 my $signal = $stat & 127;
747 if ($ec || $signal) {
748 die "$comp failed - wrong exit status $ec" .
749 ($signal ?
" (signal $signal)\n" : "\n");
754 sub _get_task_devlist
{
758 foreach my $di (@{$task->{disks
}}) {
759 if ($di->{type
} eq 'block' || $di->{type
} eq 'file') {
760 $devlist .= ',' if $devlist;
761 $devlist .= $di->{qmdevice
};
763 die "implement me (type '$di->{type}')";
770 my ($self, $task, $vmid) = @_;
771 return if !$self->{vmlist
}->{$vmid}->{agent
} || $task->{mode
} eq 'stop' || !$self->{vm_was_running
};
773 if (!PVE
::QemuServer
::qga_check_running
($vmid, 1)) {
774 $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?");
778 $self->loginfo("issuing guest-agent 'fs-freeze' command");
779 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze") };
780 $self->logerr($@) if $@;
782 return 1; # even on mon command error, ensure we always thaw again
785 # only call if fs_freeze return 1
787 my ($self, $vmid) = @_;
789 $self->loginfo("issuing guest-agent 'fs-thaw' command");
790 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw") };
791 $self->logerr($@) if $@;
794 # we need a running QEMU/KVM process for backup, starts a paused (prelaunch)
795 # one if VM isn't already running
796 sub enforce_vm_running_for_backup
{
797 my ($self, $vmid) = @_;
799 if (PVE
::QemuServer
::check_running
($vmid)) {
800 $self->{vm_was_running
} = 1;
805 $self->loginfo("starting kvm to execute backup task");
806 # start with skiplock
811 PVE
::QemuServer
::vm_start
($self->{storecfg
}, $vmid, $params);
816 # resume VM againe once we got in a clear state (stop mode backup of running VM)
817 sub resume_vm_after_job_start
{
818 my ($self, $task, $vmid) = @_;
820 return if !$self->{vm_was_running
};
822 if (my $stoptime = $task->{vmstoptime
}) {
823 my $delay = time() - $task->{vmstoptime
};
824 $task->{vmstoptime
} = undef; # avoid printing 'online after ..' twice
825 $self->loginfo("resuming VM again after $delay seconds");
827 $self->loginfo("resuming VM again");
829 mon_cmd
($vmid, 'cont');
832 # stop again if VM was not running before
833 sub restore_vm_power_state
{
834 my ($self, $vmid) = @_;
836 # we always let VMs keep running
837 return if $self->{vm_was_running
};
840 my $resp = mon_cmd
($vmid, 'query-status');
841 my $status = $resp && $resp->{status
} ?
$resp->{status
} : 'unknown';
842 if ($status eq 'prelaunch') {
843 $self->loginfo("stopping kvm after backup task");
844 PVE
::QemuServer
::vm_stop
($self->{storecfg
}, $vmid, 1);
846 $self->loginfo("kvm status changed after backup ('$status') - keep VM running");
852 sub mon_backup_cancel
{
853 my ($self, $vmid) = @_;
855 $self->loginfo("aborting backup job");
856 eval { mon_cmd
($vmid, 'backup-cancel') };
857 $self->logerr($@) if $@;
861 my ($self, $task, $vmid) = @_;
867 my ($self, $task, $vmid) = @_;