PVE/VZDump/QemuServer.pm

   1 package PVE::VZDump::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use IO::File;
   9 use IPC::Open3;
  10 use JSON;
  11 use POSIX qw(EINTR EAGAIN);
  12
  13 use PVE::Cluster qw(cfs_read_file);
  14 use PVE::INotify;
  15 use PVE::IPCC;
  16 use PVE::JSONSchema;
  17 use PVE::PBSClient;
  18 use PVE::RESTEnvironment qw(log_warn);
  19 use PVE::QMPClient;
  20 use PVE::Storage::Plugin;
  21 use PVE::Storage::PBSPlugin;
  22 use PVE::Storage;
  23 use PVE::Tools;
  24 use PVE::VZDump;
  25 use PVE::Format qw(render_duration render_bytes);
  26
  27 use PVE::QemuConfig;
  28 use PVE::QemuServer;
  29 use PVE::QemuServer::Machine;
  30 use PVE::QemuServer::Monitor qw(mon_cmd);
  31
  32 use base qw (PVE::VZDump::Plugin);
  33
  34 sub new {
  35     my ($class, $vzdump) = @_;
  36
  37     PVE::VZDump::check_bin('qm');
  38
  39     my $self = bless { vzdump => $vzdump }, $class;
  40
  41     $self->{vmlist} = PVE::QemuServer::vzlist();
  42     $self->{storecfg} = PVE::Storage::config();
  43
  44     return $self;
  45 };
  46
  47 sub type {
  48     return 'qemu';
  49 }
  50
  51 sub vmlist {
  52     my ($self) = @_;
  53     return [ keys %{$self->{vmlist}} ];
  54 }
  55
  56 sub prepare {
  57     my ($self, $task, $vmid, $mode) = @_;
  58
  59     $task->{disks} = [];
  60
  61     my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid);
  62
  63     $self->loginfo("VM Name: $conf->{name}")
  64         if defined($conf->{name});
  65
  66     $self->{vm_was_running} = 1;
  67     $self->{vm_was_paused} = 0;
  68     if (!PVE::QemuServer::check_running($vmid)) {
  69         $self->{vm_was_running} = 0;
  70     } elsif (PVE::QemuServer::vm_is_paused($vmid)) {
  71         $self->{vm_was_paused} = 1;
  72     }
  73
  74     $task->{hostname} = $conf->{name};
  75
  76     my $hostname = PVE::INotify::nodename();
  77
  78     my $vollist = [];
  79     my $drivehash = {};
  80     my $backup_volumes = PVE::QemuConfig->get_backup_volumes($conf);
  81
  82     foreach my $volume (@{$backup_volumes}) {
  83         my $name = $volume->{key};
  84         my $volume_config = $volume->{volume_config};
  85         my $volid = $volume_config->{file};
  86
  87         if (!$volume->{included}) {
  88             $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})");
  89             next;
  90         } elsif ($self->{vm_was_running} && $volume_config->{iothread} &&
  91                  !PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
  92             die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " .
  93                 "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
  94         } else {
  95             my $log = "include disk '$name' '$volid'";
  96             if (defined(my $size = $volume_config->{size})) {
  97                 my $readable_size = PVE::JSONSchema::format_size($size);
  98                 $log .= " $readable_size";
  99             }
 100             $self->loginfo($log);
 101         }
 102
 103         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 104         push @$vollist, $volid if $storeid;
 105         $drivehash->{$name} = $volume->{volume_config};
 106     }
 107
 108     PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
 109
 110     foreach my $ds (sort keys %$drivehash) {
 111         my $drive = $drivehash->{$ds};
 112
 113         my $volid = $drive->{file};
 114         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 115
 116         my $path = $volid;
 117         if ($storeid) {
 118             $path = PVE::Storage::path($self->{storecfg}, $volid);
 119         }
 120         next if !$path;
 121
 122         my ($size, $format);
 123         if ($storeid) {
 124             # The call in list context can be expensive for certain plugins like RBD, just get size
 125             $size = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) };
 126             die "cannot determine size of volume '$volid' - $@\n" if $@;
 127
 128             my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
 129             $format = PVE::QemuServer::qemu_img_format($scfg, $volname);
 130         } else {
 131             ($size, $format) = eval {
 132                 PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5);
 133             };
 134             die "cannot determine size and format of volume '$volid' - $@\n" if $@;
 135         }
 136
 137         my $diskinfo = {
 138             path => $path,
 139             volid => $volid,
 140             storeid => $storeid,
 141             format => $format,
 142             virtdev => $ds,
 143             qmdevice => "drive-$ds",
 144         };
 145
 146         if ($ds eq 'tpmstate0') {
 147             # TPM drive only exists for backup, which is reflected in the name
 148             $diskinfo->{qmdevice} = 'drive-tpmstate0-backup';
 149             $task->{tpmpath} = $path;
 150         }
 151
 152         if (-b $path) {
 153             $diskinfo->{type} = 'block';
 154         } else {
 155             $diskinfo->{type} = 'file';
 156         }
 157
 158         push @{$task->{disks}}, $diskinfo;
 159     }
 160 }
 161
 162 sub vm_status {
 163     my ($self, $vmid) = @_;
 164
 165     my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0;
 166
 167     return wantarray ? ($running, $running ? 'running' : 'stopped') : $running;
 168 }
 169
 170 sub lock_vm {
 171     my ($self, $vmid) = @_;
 172
 173     PVE::QemuConfig->set_lock($vmid, 'backup');
 174 }
 175
 176 sub unlock_vm {
 177     my ($self, $vmid) = @_;
 178
 179     PVE::QemuConfig->remove_lock($vmid, 'backup');
 180 }
 181
 182 sub stop_vm {
 183     my ($self, $task, $vmid) = @_;
 184
 185     my $opts = $self->{vzdump}->{opts};
 186
 187     my $wait = $opts->{stopwait} * 60;
 188     # send shutdown and wait
 189     $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
 190 }
 191
 192 sub start_vm {
 193     my ($self, $task, $vmid) = @_;
 194
 195     $self->cmd ("qm start $vmid --skiplock");
 196 }
 197
 198 sub suspend_vm {
 199     my ($self, $task, $vmid) = @_;
 200
 201     return if $self->{vm_was_paused};
 202
 203     $self->cmd ("qm suspend $vmid --skiplock");
 204 }
 205
 206 sub resume_vm {
 207     my ($self, $task, $vmid) = @_;
 208
 209     return if $self->{vm_was_paused};
 210
 211     $self->cmd ("qm resume $vmid --skiplock");
 212 }
 213
 214 sub assemble {
 215     my ($self, $task, $vmid) = @_;
 216
 217     my $conffile = PVE::QemuConfig->config_file($vmid);
 218
 219     my $outfile = "$task->{tmpdir}/qemu-server.conf";
 220     my $firewall_src = "/etc/pve/firewall/$vmid.fw";
 221     my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
 222
 223     my $outfd = IO::File->new (">$outfile") ||
 224         die "unable to open '$outfile'";
 225     my $conffd = IO::File->new ($conffile, 'r') ||
 226         die "unable open '$conffile'";
 227
 228     my $found_snapshot;
 229     my $found_pending;
 230     my $found_cloudinit;
 231     while (defined (my $line = <$conffd>)) {
 232         next if $line =~ m/^\#vzdump\#/; # just to be sure
 233         next if $line =~ m/^\#qmdump\#/; # just to be sure
 234         if ($line =~ m/^\[(.*)\]\s*$/) {
 235             if ($1 =~ m/PENDING/i) {
 236                 $found_pending = 1;
 237             } elsif ($1 =~ m/special:cloudinit/) {
 238                 $found_cloudinit = 1;
 239             } else {
 240                 $found_snapshot = 1;
 241             }
 242         }
 243         next if $found_snapshot || $found_pending || $found_cloudinit; # skip all snapshots,pending changes and cloudinit config data
 244
 245         if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
 246             $self->loginfo("skip unused drive '$1' (not included into backup)");
 247             next;
 248         }
 249         next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
 250
 251         print $outfd $line;
 252     }
 253
 254     foreach my $di (@{$task->{disks}}) {
 255         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 256             my $storeid = $di->{storeid} || '';
 257             my $format = $di->{format} || '';
 258             print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
 259         } else {
 260             die "internal error";
 261         }
 262     }
 263
 264     if ($found_snapshot) {
 265         $self->loginfo("snapshots found (not included into backup)");
 266     }
 267     if ($found_pending) {
 268         $self->loginfo("pending configuration changes found (not included into backup)");
 269     }
 270
 271     PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src;
 272 }
 273
 274 sub archive {
 275     my ($self, $task, $vmid, $filename, $comp) = @_;
 276
 277     my $opts = $self->{vzdump}->{opts};
 278     my $scfg = $opts->{scfg};
 279
 280     if ($self->{vzdump}->{opts}->{pbs}) {
 281         $self->archive_pbs($task, $vmid);
 282     } else {
 283         $self->archive_vma($task, $vmid, $filename, $comp);
 284     }
 285 }
 286
 287 my $bitmap_action_to_human = sub {
 288     my ($self, $info) = @_;
 289
 290     my $action = $info->{action};
 291
 292     if ($action eq "not-used") {
 293         return "disabled (no support)";
 294     } elsif ($action eq "not-used-removed") {
 295         return "disabled (old bitmap cleared)";
 296     } elsif ($action eq "new") {
 297         return "created new";
 298     } elsif ($action eq "used") {
 299         if ($info->{dirty} == 0) {
 300             return "OK (drive clean)";
 301         } else {
 302             my $size = render_bytes($info->{size}, 1);
 303             my $dirty = render_bytes($info->{dirty}, 1);
 304             return "OK ($dirty of $size dirty)";
 305         }
 306     } elsif ($action eq "invalid") {
 307         return "existing bitmap was invalid and has been cleared";
 308     } else {
 309         return "unknown";
 310     }
 311 };
 312
 313 my $query_backup_status_loop = sub {
 314     my ($self, $vmid, $job_uuid, $qemu_support) = @_;
 315
 316     my $starttime = time ();
 317     my $last_time = $starttime;
 318     my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0);
 319     my ($transferred, $reused);
 320
 321     my $get_mbps = sub {
 322         my ($mb, $delta) = @_;
 323         return "0 B/s" if $mb <= 0;
 324         my $bw = int(($mb / $delta));
 325         return render_bytes($bw, 1) . "/s";
 326     };
 327
 328     my $target = 0;
 329     my $last_reused = 0;
 330     my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'};
 331     my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 332     if ($has_query_bitmap) {
 333         my $total = 0;
 334         my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info');
 335         for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) {
 336             if (!$is_template) {
 337                 my $text = $bitmap_action_to_human->($self, $info);
 338                 my $drive = $info->{drive};
 339                 $drive =~ s/^drive-//; # for consistency
 340                 $self->loginfo("$drive: dirty-bitmap status: $text");
 341             }
 342             $target += $info->{dirty};
 343             $total += $info->{size};
 344             $last_reused += $info->{size} - $info->{dirty};
 345         }
 346         if ($target < $total) {
 347             my $total_h = render_bytes($total, 1);
 348             my $target_h = render_bytes($target, 1);
 349             $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
 350         }
 351     }
 352
 353     my $last_finishing = 0;
 354     while(1) {
 355         my $status = mon_cmd($vmid, 'query-backup');
 356
 357         my $total = $status->{total} || 0;
 358         my $dirty = $status->{dirty};
 359         $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap;
 360         $transferred = $status->{transferred} || 0;
 361         $reused = $status->{reused};
 362         my $percent = $target ? int(($transferred * 100)/$target) : 100;
 363         my $zero = $status->{'zero-bytes'} || 0;
 364
 365         die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $job_uuid);
 366
 367         my $ctime = time();
 368         my $duration = $ctime - $starttime;
 369
 370         my $rbytes = $transferred - $last_transferred;
 371         my $wbytes;
 372         if ($reused) {
 373             # reused includes zero bytes for PBS
 374             $wbytes = $rbytes - ($reused - $last_reused);
 375         } else {
 376             $wbytes = $rbytes - ($zero - $last_zero);
 377         }
 378
 379         my $timediff = ($ctime - $last_time) || 1; # fixme
 380         my $mbps_read = $get_mbps->($rbytes, $timediff);
 381         my $mbps_write = $get_mbps->($wbytes, $timediff);
 382         my $target_h = render_bytes($target, 1);
 383         my $transferred_h = render_bytes($transferred, 1);
 384
 385         my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s"
 386             .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration));
 387
 388         my $res = $status->{status} || 'unknown';
 389         if ($res ne 'active') {
 390             if ($last_percent < 100) {
 391                 $self->loginfo($statusline);
 392             }
 393             if ($res ne 'done') {
 394                 die (($status->{errmsg} || "unknown error") . "\n") if $res eq 'error';
 395                 die "got unexpected status '$res'\n";
 396             }
 397             $last_target = $target if $target;
 398             $last_total = $total if $total;
 399             $last_zero = $zero if $zero;
 400             $last_transferred = $transferred if $transferred;
 401             last;
 402         }
 403         if ($percent != $last_percent && ($timediff > 2)) {
 404             $self->loginfo($statusline);
 405             $last_percent = $percent;
 406             $last_target = $target if $target;
 407             $last_total = $total if $total;
 408             $last_zero = $zero if $zero;
 409             $last_transferred = $transferred if $transferred;
 410             $last_time = $ctime;
 411             $last_reused = $reused;
 412
 413             if (!$last_finishing && $status->{finishing}) {
 414                 $self->loginfo("Waiting for server to finish backup validation...");
 415             }
 416             $last_finishing = $status->{finishing};
 417         }
 418         sleep(1);
 419     }
 420
 421     my $duration = time() - $starttime;
 422
 423     if ($last_zero) {
 424         my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0;
 425         my $zero_h = render_bytes($last_zero);
 426         $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
 427     }
 428     if ($reused) {
 429         my $reused_h = render_bytes($reused);
 430         my $reuse_per = int($reused * 100 / $last_total);
 431         $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
 432     }
 433     if ($transferred) {
 434         my $transferred_h = render_bytes($transferred);
 435         if ($duration) {
 436             my $mbps = $get_mbps->($transferred, $duration);
 437             $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
 438         } else {
 439             $self->loginfo("transferred $transferred_h in <1 seconds");
 440         }
 441     }
 442
 443     return {
 444         total => $last_total,
 445         reused => $reused,
 446     };
 447 };
 448
 449 my $attach_tpmstate_drive = sub {
 450     my ($self, $task, $vmid) = @_;
 451
 452     return if !$task->{tpmpath};
 453
 454     # unconditionally try to remove the tpmstate-named drive - it only exists
 455     # for backing up, and avoids errors if left over from some previous event
 456     eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
 457
 458     $self->loginfo('attaching TPM drive to QEMU for backup');
 459
 460     my $drive = "file=$task->{tpmpath},if=none,read-only=on,id=drive-tpmstate0-backup";
 461     $drive =~ s/\\/\\\\/g;
 462     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
 463     die "attaching TPM drive failed - $ret\n" if $ret !~ m/OK/s;
 464 };
 465
 466 my $detach_tpmstate_drive = sub {
 467     my ($task, $vmid) = @_;
 468     return if !$task->{tpmpath} || !PVE::QemuServer::check_running($vmid);
 469     eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
 470 };
 471
 472 my sub add_backup_performance_options {
 473     my ($qmp_param, $perf, $qemu_support) = @_;
 474
 475     return if !$perf || scalar(keys $perf->%*) == 0;
 476
 477     if (!$qemu_support) {
 478         my $settings_string = join(', ', sort keys $perf->%*);
 479         log_warn("ignoring setting(s): $settings_string - issue checking if supported");
 480         return;
 481     }
 482
 483     if (defined($perf->{'max-workers'})) {
 484         if ($qemu_support->{'backup-max-workers'}) {
 485             $qmp_param->{'max-workers'} = int($perf->{'max-workers'});
 486         } else {
 487             log_warn("ignoring 'max-workers' setting - not supported by running QEMU");
 488         }
 489     }
 490 }
 491
 492 sub archive_pbs {
 493     my ($self, $task, $vmid) = @_;
 494
 495     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 496     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 497
 498     my $opts = $self->{vzdump}->{opts};
 499     my $scfg = $opts->{scfg};
 500
 501     my $starttime = time();
 502
 503     my $fingerprint = $scfg->{fingerprint};
 504     my $repo = PVE::PBSClient::get_repository($scfg);
 505     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage});
 506     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage});
 507     my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage});
 508
 509     my $diskcount = scalar(@{$task->{disks}});
 510     # proxmox-backup-client can only handle raw files and block devs
 511     # only use it (directly) for disk-less VMs
 512     if (!$diskcount) {
 513         my @pathlist;
 514         $self->loginfo("backup contains no disks");
 515
 516         local $ENV{PBS_PASSWORD} = $password;
 517         local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
 518         my $cmd = [
 519             '/usr/bin/proxmox-backup-client',
 520             'backup',
 521             '--repository', $repo,
 522             '--backup-type', 'vm',
 523             '--backup-id', "$vmid",
 524             '--backup-time', $task->{backup_time},
 525         ];
 526         if (defined(my $ns = $scfg->{namespace})) {
 527             push @$cmd, '--ns', $ns;
 528         }
 529         if (-e $keyfile) {
 530             $self->loginfo("enabling encryption");
 531             push @$cmd, '--keyfile', $keyfile;
 532             if (defined($master_keyfile)) {
 533                 if (-e $master_keyfile) {
 534                     $self->loginfo("enabling master key feature");
 535                     push @$cmd, '--master-pubkey-file', $master_keyfile;
 536                 } elsif ($scfg->{'master-pubkey'}) {
 537                     die "master public key configured but no key file found\n";
 538                 }
 539             }
 540         } else {
 541             my $encryption_fp = $scfg->{'encryption-key'};
 542             die "encryption configured ('$encryption_fp') but no encryption key file found!\n"
 543                 if $encryption_fp;
 544             $self->loginfo("WARNING: backup target is configured with master key, but this backup is not encrypted - master key settings will be ignored!")
 545                 if defined($master_keyfile) && -e $master_keyfile;
 546         }
 547
 548         push @$cmd, "qemu-server.conf:$conffile";
 549         push @$cmd, "fw.conf:$firewall" if -e $firewall;
 550
 551         $self->loginfo("starting template backup");
 552         $self->loginfo(join(' ', @$cmd));
 553
 554         $self->cmd($cmd);
 555
 556         return;
 557     }
 558
 559     # get list early so we die on unkown drive types before doing anything
 560     my $devlist = _get_task_devlist($task);
 561
 562     $self->enforce_vm_running_for_backup($vmid);
 563     $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
 564
 565     my $backup_job_uuid;
 566     eval {
 567         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 568             die "interrupted by signal\n";
 569         };
 570
 571         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
 572         my $err = $@;
 573         if (!$qemu_support || $err) {
 574             die "query-proxmox-support returned empty value\n" if !$err;
 575             if ($err =~ m/The command query-proxmox-support has not been found/) {
 576                 die "PBS backups are not supported by the running QEMU version. Please make "
 577                   . "sure you've installed the latest version and the VM has been restarted.\n";
 578             } else {
 579                 die "QMP command query-proxmox-support failed - $err\n";
 580             }
 581         }
 582
 583         if (!defined($qemu_support->{"pbs-masterkey"}) && -e $master_keyfile) {
 584             $self->loginfo("WARNING: backup target is configured with master key, but running QEMU version does not support master keys.");
 585             $self->loginfo("Please make sure you've installed the latest version and the VM has been restarted to use master key feature.");
 586             $master_keyfile = undef; # skip rest of master key handling below
 587         }
 588
 589         $attach_tpmstate_drive->($self, $task, $vmid);
 590
 591         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 592
 593         my $params = {
 594             format => "pbs",
 595             'backup-file' => $repo,
 596             'backup-id' => "$vmid",
 597             'backup-time' => $task->{backup_time},
 598             password => $password,
 599             devlist => $devlist,
 600             'config-file' => $conffile,
 601         };
 602         if (defined(my $ns = $scfg->{namespace})) {
 603             $params->{'backup-ns'} = $ns;
 604         }
 605
 606         $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit};
 607         add_backup_performance_options($params, $opts->{performance}, $qemu_support);
 608
 609         $params->{fingerprint} = $fingerprint if defined($fingerprint);
 610         $params->{'firewall-file'} = $firewall if -e $firewall;
 611         if (-e $keyfile) {
 612             $self->loginfo("enabling encryption");
 613             $params->{keyfile} = $keyfile;
 614             $params->{encrypt} = JSON::true;
 615             if (defined($master_keyfile)) {
 616                 if (-e $master_keyfile) {
 617                     $self->loginfo("enabling master key feature");
 618                     $params->{"master-keyfile"} = $master_keyfile;
 619                 } elsif ($scfg->{'master-pubkey'}) {
 620                     die "master public key configured but no key file found\n";
 621                 }
 622             }
 623         } else {
 624             my $encryption_fp = $scfg->{'encryption-key'};
 625             die "encryption configured ('$encryption_fp') but no encryption key file found!\n"
 626                 if $encryption_fp;
 627             $self->loginfo("WARNING: backup target is configured with master key, but this backup is not encrypted - master key settings will be ignored!")
 628                 if defined($master_keyfile) && -e $master_keyfile;
 629             $params->{encrypt} = JSON::false;
 630         }
 631
 632         my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 633         $params->{'use-dirty-bitmap'} = JSON::true
 634             if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template;
 635
 636         $params->{timeout} = 125; # give some time to connect to the backup server
 637
 638         my $res = eval { mon_cmd($vmid, "backup", %$params) };
 639         my $qmperr = $@;
 640         $backup_job_uuid = $res->{UUID} if $res;
 641
 642         if ($fs_frozen) {
 643             $self->qga_fs_thaw($vmid);
 644         }
 645
 646         die $qmperr if $qmperr;
 647         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 648
 649         $self->loginfo("started backup task '$backup_job_uuid'");
 650
 651         $self->resume_vm_after_job_start($task, $vmid);
 652
 653         my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support);
 654         $task->{size} = $stat->{total};
 655     };
 656     my $err = $@;
 657     if ($err) {
 658         $self->logerr($err);
 659         $self->mon_backup_cancel($vmid);
 660         $self->resume_vm_after_job_start($task, $vmid);
 661     }
 662     $self->restore_vm_power_state($vmid);
 663
 664     die $err if $err;
 665 }
 666
 667 my $fork_compressor_pipe = sub {
 668     my ($self, $comp, $outfileno) = @_;
 669
 670     my @pipefd = POSIX::pipe();
 671     my $cpid = fork();
 672     die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0;
 673     if ($cpid == 0) {
 674         eval {
 675             POSIX::close($pipefd[1]);
 676             # redirect STDIN
 677             my $fd = fileno(STDIN);
 678             close STDIN;
 679             POSIX::close(0) if $fd != 0;
 680             die "unable to redirect STDIN - $!"
 681                 if !open(STDIN, "<&", $pipefd[0]);
 682
 683             # redirect STDOUT
 684             $fd = fileno(STDOUT);
 685             close STDOUT;
 686             POSIX::close (1) if $fd != 1;
 687
 688             die "unable to redirect STDOUT - $!"
 689                 if !open(STDOUT, ">&", $outfileno);
 690
 691             exec($comp);
 692             die "fork compressor '$comp' failed\n";
 693         };
 694         if (my $err = $@) {
 695             $self->logerr($err);
 696             POSIX::_exit(1);
 697         }
 698         POSIX::_exit(0);
 699         kill(-9, $$);
 700     } else {
 701         POSIX::close($pipefd[0]);
 702         $outfileno = $pipefd[1];
 703     }
 704
 705     return ($cpid, $outfileno);
 706 };
 707
 708 sub archive_vma {
 709     my ($self, $task, $vmid, $filename, $comp) = @_;
 710
 711     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 712     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 713
 714     my $opts = $self->{vzdump}->{opts};
 715
 716     my $starttime = time();
 717
 718     my $speed = 0;
 719     if ($opts->{bwlimit}) {
 720         $speed = $opts->{bwlimit}*1024;
 721     }
 722
 723     my $diskcount = scalar(@{$task->{disks}});
 724     if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) {
 725         my @pathlist;
 726         foreach my $di (@{$task->{disks}}) {
 727             if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 728                 push @pathlist, "$di->{qmdevice}=$di->{path}";
 729             } else {
 730                 die "implement me";
 731             }
 732         }
 733
 734         if (!$diskcount) {
 735             $self->loginfo("backup contains no disks");
 736         }
 737
 738         my $outcmd;
 739         if ($comp) {
 740             $outcmd = "exec:$comp";
 741         } else {
 742             $outcmd = "exec:cat";
 743         }
 744
 745         $outcmd .= " > $filename" if !$opts->{stdout};
 746
 747         my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
 748         push @$cmd, '-c', $firewall if -e $firewall;
 749         push @$cmd, $outcmd, @pathlist;
 750
 751         $self->loginfo("starting template backup");
 752         $self->loginfo(join(' ', @$cmd));
 753
 754         if ($opts->{stdout}) {
 755             $self->cmd($cmd, output => ">&" . fileno($opts->{stdout}));
 756         } else {
 757             $self->cmd($cmd);
 758         }
 759
 760         return;
 761     }
 762
 763     my $devlist = _get_task_devlist($task);
 764
 765     $self->enforce_vm_running_for_backup($vmid);
 766     $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
 767
 768     my $cpid;
 769     my $backup_job_uuid;
 770
 771     eval {
 772         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 773             die "interrupted by signal\n";
 774         };
 775
 776         # Currently, failing to determine Proxmox support is not critical here, because it's only
 777         # used for performance settings like 'max-workers'.
 778         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
 779         log_warn($@) if $@;
 780
 781         $attach_tpmstate_drive->($self, $task, $vmid);
 782
 783         my $outfh;
 784         if ($opts->{stdout}) {
 785             $outfh = $opts->{stdout};
 786         } else {
 787             $outfh = IO::File->new($filename, "w") ||
 788                 die "unable to open file '$filename' - $!\n";
 789         }
 790         my $outfileno = fileno($outfh);
 791
 792         if ($comp) {
 793             ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno);
 794         }
 795
 796         my $qmpclient = PVE::QMPClient->new();
 797         my $backup_cb = sub {
 798             my ($vmid, $resp) = @_;
 799             $backup_job_uuid = $resp->{return}->{UUID};
 800         };
 801         my $add_fd_cb = sub {
 802             my ($vmid, $resp) = @_;
 803
 804             my $params = {
 805                 'backup-file' => "/dev/fdname/backup",
 806                 speed => $speed,
 807                 'config-file' => $conffile,
 808                 devlist => $devlist
 809             };
 810             $params->{'firewall-file'} = $firewall if -e $firewall;
 811             add_backup_performance_options($params, $opts->{performance}, $qemu_support);
 812
 813             $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
 814         };
 815
 816         $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd', fd => $outfileno, fdname => "backup");
 817
 818         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 819
 820         eval { $qmpclient->queue_execute(30) };
 821         my $qmperr = $@;
 822
 823         if ($fs_frozen) {
 824             $self->qga_fs_thaw($vmid);
 825         }
 826
 827         die $qmperr if $qmperr;
 828         die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid};
 829
 830         if ($cpid) {
 831             POSIX::close($outfileno) == 0 ||
 832                 die "close output file handle failed\n";
 833         }
 834
 835         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 836
 837         $self->loginfo("started backup task '$backup_job_uuid'");
 838
 839         $self->resume_vm_after_job_start($task, $vmid);
 840
 841         $query_backup_status_loop->($self, $vmid, $backup_job_uuid);
 842     };
 843     my $err = $@;
 844     if ($err) {
 845         $self->logerr($err);
 846         $self->mon_backup_cancel($vmid);
 847         $self->resume_vm_after_job_start($task, $vmid);
 848     }
 849
 850     $self->restore_vm_power_state($vmid);
 851
 852     if ($err) {
 853         if ($cpid) {
 854             kill(9, $cpid);
 855             waitpid($cpid, 0);
 856         }
 857         die $err;
 858     }
 859
 860     if ($cpid && (waitpid($cpid, 0) > 0)) {
 861         my $stat = $?;
 862         my $ec = $stat >> 8;
 863         my $signal = $stat & 127;
 864         if ($ec || $signal) {
 865             die "$comp failed - wrong exit status $ec" .
 866                 ($signal ? " (signal $signal)\n" : "\n");
 867         }
 868     }
 869 }
 870
 871 sub _get_task_devlist {
 872     my ($task) = @_;
 873
 874     my $devlist = '';
 875     foreach my $di (@{$task->{disks}}) {
 876         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 877             $devlist .= ',' if $devlist;
 878             $devlist .= $di->{qmdevice};
 879         } else {
 880             die "implement me (type '$di->{type}')";
 881         }
 882     }
 883     return $devlist;
 884 }
 885
 886 sub qga_fs_freeze {
 887     my ($self, $task, $vmid) = @_;
 888     return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused};
 889
 890     if (!PVE::QemuServer::qga_check_running($vmid, 1)) {
 891         $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?");
 892         return;
 893     }
 894
 895     my $freeze = PVE::QemuServer::get_qga_key($self->{vmlist}->{$vmid}, 'freeze-fs-on-backup') // 1;
 896     if (!$freeze) {
 897         $self->loginfo("skipping guest-agent 'fs-freeze', disabled in VM options");
 898         return;
 899     }
 900
 901     $self->loginfo("issuing guest-agent 'fs-freeze' command");
 902     eval { mon_cmd($vmid, "guest-fsfreeze-freeze") };
 903     $self->logerr($@) if $@;
 904
 905     return 1; # even on mon command error, ensure we always thaw again
 906 }
 907
 908 # only call if fs_freeze return 1
 909 sub qga_fs_thaw {
 910     my ($self, $vmid) = @_;
 911
 912     $self->loginfo("issuing guest-agent 'fs-thaw' command");
 913     eval { mon_cmd($vmid, "guest-fsfreeze-thaw") };
 914     $self->logerr($@) if $@;
 915 }
 916
 917 # we need a running QEMU/KVM process for backup, starts a paused (prelaunch)
 918 # one if VM isn't already running
 919 sub enforce_vm_running_for_backup {
 920     my ($self, $vmid) = @_;
 921
 922     if (PVE::QemuServer::check_running($vmid)) {
 923         $self->{vm_was_running} = 1;
 924         return;
 925     }
 926
 927     eval {
 928         $self->loginfo("starting kvm to execute backup task");
 929         # start with skiplock
 930         my $params = {
 931             skiplock => 1,
 932             skiptemplate => 1,
 933             paused => 1,
 934         };
 935         PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params);
 936     };
 937     die $@ if $@;
 938 }
 939
 940 # resume VM again once in a clear state (stop mode backup of running VM)
 941 sub resume_vm_after_job_start {
 942     my ($self, $task, $vmid) = @_;
 943
 944     return if !$self->{vm_was_running} || $self->{vm_was_paused};
 945
 946     if (my $stoptime = $task->{vmstoptime}) {
 947         my $delay = time() - $task->{vmstoptime};
 948         $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice
 949         $self->loginfo("resuming VM again after $delay seconds");
 950     } else {
 951         $self->loginfo("resuming VM again");
 952     }
 953     mon_cmd($vmid, 'cont', timeout => 45);
 954 }
 955
 956 # stop again if VM was not running before
 957 sub restore_vm_power_state {
 958     my ($self, $vmid) = @_;
 959
 960     # we always let VMs keep running
 961     return if $self->{vm_was_running};
 962
 963     eval {
 964         my $resp = mon_cmd($vmid, 'query-status');
 965         my $status = $resp && $resp->{status} ?  $resp->{status} : 'unknown';
 966         if ($status eq 'prelaunch') {
 967             $self->loginfo("stopping kvm after backup task");
 968             PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1);
 969         } else {
 970             $self->loginfo("kvm status changed after backup ('$status') - keep VM running");
 971         }
 972     };
 973     warn $@ if $@;
 974 }
 975
 976 sub mon_backup_cancel {
 977     my ($self, $vmid) = @_;
 978
 979     $self->loginfo("aborting backup job");
 980     eval { mon_cmd($vmid, 'backup-cancel') };
 981     $self->logerr($@) if $@;
 982 }
 983
 984 sub snapshot {
 985     my ($self, $task, $vmid) = @_;
 986
 987     # nothing to do
 988 }
 989
 990 sub cleanup {
 991     my ($self, $task, $vmid) = @_;
 992
 993     $detach_tpmstate_drive->($task, $vmid);
 994
 995     if ($self->{qmeventd_fh}) {
 996         close($self->{qmeventd_fh});
 997     }
 998 }
 999
1000 1;