PVE/VZDump/QemuServer.pm

   1 package PVE::VZDump::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use IO::File;
   9 use IPC::Open3;
  10 use JSON;
  11 use POSIX qw(EINTR EAGAIN);
  12
  13 use PVE::Cluster qw(cfs_read_file);
  14 use PVE::INotify;
  15 use PVE::IPCC;
  16 use PVE::JSONSchema;
  17 use PVE::PBSClient;
  18 use PVE::RESTEnvironment qw(log_warn);
  19 use PVE::QMPClient;
  20 use PVE::Storage::Plugin;
  21 use PVE::Storage::PBSPlugin;
  22 use PVE::Storage;
  23 use PVE::Tools;
  24 use PVE::VZDump;
  25 use PVE::Format qw(render_duration render_bytes);
  26
  27 use PVE::QemuConfig;
  28 use PVE::QemuServer;
  29 use PVE::QemuServer::Helpers;
  30 use PVE::QemuServer::Machine;
  31 use PVE::QemuServer::Monitor qw(mon_cmd);
  32
  33 use base qw (PVE::VZDump::Plugin);
  34
  35 sub new {
  36     my ($class, $vzdump) = @_;
  37
  38     PVE::VZDump::check_bin('qm');
  39
  40     my $self = bless { vzdump => $vzdump }, $class;
  41
  42     $self->{vmlist} = PVE::QemuServer::vzlist();
  43     $self->{storecfg} = PVE::Storage::config();
  44
  45     return $self;
  46 };
  47
  48 sub type {
  49     return 'qemu';
  50 }
  51
  52 sub vmlist {
  53     my ($self) = @_;
  54     return [ keys %{$self->{vmlist}} ];
  55 }
  56
  57 sub prepare {
  58     my ($self, $task, $vmid, $mode) = @_;
  59
  60     $task->{disks} = [];
  61
  62     my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid);
  63
  64     $self->loginfo("VM Name: $conf->{name}")
  65         if defined($conf->{name});
  66
  67     $self->{vm_was_running} = 1;
  68     $self->{vm_was_paused} = 0;
  69     if (!PVE::QemuServer::check_running($vmid)) {
  70         $self->{vm_was_running} = 0;
  71     } elsif (PVE::QemuServer::vm_is_paused($vmid, 0)) {
  72         # Do not treat a suspended VM as paused, as it would cause us to skip
  73         # fs-freeze even if the VM wakes up before we reach qga_fs_freeze.
  74         $self->{vm_was_paused} = 1;
  75     }
  76
  77     $task->{hostname} = $conf->{name};
  78
  79     my $hostname = PVE::INotify::nodename();
  80
  81     my $vollist = [];
  82     my $drivehash = {};
  83     my $backup_volumes = PVE::QemuConfig->get_backup_volumes($conf);
  84
  85     foreach my $volume (@{$backup_volumes}) {
  86         my $name = $volume->{key};
  87         my $volume_config = $volume->{volume_config};
  88         my $volid = $volume_config->{file};
  89
  90         if (!$volume->{included}) {
  91             $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})");
  92             next;
  93         } elsif ($self->{vm_was_running} && $volume_config->{iothread} &&
  94                  !PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
  95             die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " .
  96                 "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
  97         } else {
  98             my $log = "include disk '$name' '$volid'";
  99             if (defined(my $size = $volume_config->{size})) {
 100                 my $readable_size = PVE::JSONSchema::format_size($size);
 101                 $log .= " $readable_size";
 102             }
 103             $self->loginfo($log);
 104         }
 105
 106         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 107         push @$vollist, $volid if $storeid;
 108         $drivehash->{$name} = $volume->{volume_config};
 109     }
 110
 111     PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
 112
 113     foreach my $ds (sort keys %$drivehash) {
 114         my $drive = $drivehash->{$ds};
 115
 116         my $volid = $drive->{file};
 117         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 118
 119         my $path = $volid;
 120         if ($storeid) {
 121             $path = PVE::Storage::path($self->{storecfg}, $volid);
 122         }
 123         next if !$path;
 124
 125         my ($size, $format);
 126         if ($storeid) {
 127             # The call in list context can be expensive for certain plugins like RBD, just get size
 128             $size = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) };
 129             die "cannot determine size of volume '$volid' - $@\n" if $@;
 130
 131             my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
 132             $format = PVE::QemuServer::qemu_img_format($scfg, $volname);
 133         } else {
 134             ($size, $format) = eval {
 135                 PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5);
 136             };
 137             die "cannot determine size and format of volume '$volid' - $@\n" if $@;
 138         }
 139
 140         my $diskinfo = {
 141             path => $path,
 142             volid => $volid,
 143             storeid => $storeid,
 144             size => $size,
 145             format => $format,
 146             virtdev => $ds,
 147             qmdevice => "drive-$ds",
 148         };
 149
 150         if ($ds eq 'tpmstate0') {
 151             # TPM drive only exists for backup, which is reflected in the name
 152             $diskinfo->{qmdevice} = 'drive-tpmstate0-backup';
 153             $task->{tpmpath} = $path;
 154         }
 155
 156         if (-b $path) {
 157             $diskinfo->{type} = 'block';
 158         } else {
 159             $diskinfo->{type} = 'file';
 160         }
 161
 162         push @{$task->{disks}}, $diskinfo;
 163     }
 164 }
 165
 166 sub vm_status {
 167     my ($self, $vmid) = @_;
 168
 169     my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0;
 170
 171     return wantarray ? ($running, $running ? 'running' : 'stopped') : $running;
 172 }
 173
 174 sub lock_vm {
 175     my ($self, $vmid) = @_;
 176
 177     PVE::QemuConfig->set_lock($vmid, 'backup');
 178 }
 179
 180 sub unlock_vm {
 181     my ($self, $vmid) = @_;
 182
 183     PVE::QemuConfig->remove_lock($vmid, 'backup');
 184 }
 185
 186 sub stop_vm {
 187     my ($self, $task, $vmid) = @_;
 188
 189     my $opts = $self->{vzdump}->{opts};
 190
 191     my $wait = $opts->{stopwait} * 60;
 192     # send shutdown and wait
 193     $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
 194 }
 195
 196 sub start_vm {
 197     my ($self, $task, $vmid) = @_;
 198
 199     $self->cmd ("qm start $vmid --skiplock");
 200 }
 201
 202 sub suspend_vm {
 203     my ($self, $task, $vmid) = @_;
 204
 205     return if $self->{vm_was_paused};
 206
 207     $self->cmd ("qm suspend $vmid --skiplock");
 208 }
 209
 210 sub resume_vm {
 211     my ($self, $task, $vmid) = @_;
 212
 213     return if $self->{vm_was_paused};
 214
 215     $self->cmd ("qm resume $vmid --skiplock");
 216 }
 217
 218 sub assemble {
 219     my ($self, $task, $vmid) = @_;
 220
 221     my $conffile = PVE::QemuConfig->config_file($vmid);
 222
 223     my $outfile = "$task->{tmpdir}/qemu-server.conf";
 224     my $firewall_src = "/etc/pve/firewall/$vmid.fw";
 225     my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
 226
 227     my $outfd = IO::File->new(">$outfile") or die "unable to open '$outfile' - $!\n";
 228     my $conffd = IO::File->new($conffile, 'r') or die "unable to open '$conffile' - $!\n";
 229
 230     my $found_snapshot;
 231     my $found_pending;
 232     my $found_cloudinit;
 233     while (defined (my $line = <$conffd>)) {
 234         next if $line =~ m/^\#vzdump\#/; # just to be sure
 235         next if $line =~ m/^\#qmdump\#/; # just to be sure
 236         if ($line =~ m/^\[(.*)\]\s*$/) {
 237             if ($1 =~ m/PENDING/i) {
 238                 $found_pending = 1;
 239             } elsif ($1 =~ m/special:cloudinit/) {
 240                 $found_cloudinit = 1;
 241             } else {
 242                 $found_snapshot = 1;
 243             }
 244         }
 245         next if $found_snapshot || $found_pending || $found_cloudinit; # skip all snapshots,pending changes and cloudinit config data
 246
 247         if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
 248             $self->loginfo("skip unused drive '$1' (not included into backup)");
 249             next;
 250         }
 251         next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
 252
 253         print $outfd $line;
 254     }
 255
 256     foreach my $di (@{$task->{disks}}) {
 257         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 258             my $storeid = $di->{storeid} || '';
 259             my $format = $di->{format} || '';
 260             print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
 261         } else {
 262             die "internal error";
 263         }
 264     }
 265
 266     if ($found_snapshot) {
 267         $self->loginfo("snapshots found (not included into backup)");
 268     }
 269     if ($found_pending) {
 270         $self->loginfo("pending configuration changes found (not included into backup)");
 271     }
 272
 273     PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src;
 274 }
 275
 276 sub archive {
 277     my ($self, $task, $vmid, $filename, $comp) = @_;
 278
 279     my $opts = $self->{vzdump}->{opts};
 280     my $scfg = $opts->{scfg};
 281
 282     if ($self->{vzdump}->{opts}->{pbs}) {
 283         $self->archive_pbs($task, $vmid);
 284     } else {
 285         $self->archive_vma($task, $vmid, $filename, $comp);
 286     }
 287 }
 288
 289 my $bitmap_action_to_human = sub {
 290     my ($self, $info) = @_;
 291
 292     my $action = $info->{action};
 293
 294     if ($action eq "not-used") {
 295         return "disabled (no support)";
 296     } elsif ($action eq "not-used-removed") {
 297         return "disabled (old bitmap cleared)";
 298     } elsif ($action eq "new") {
 299         return "created new";
 300     } elsif ($action eq "used") {
 301         if ($info->{dirty} == 0) {
 302             return "OK (drive clean)";
 303         } else {
 304             my $size = render_bytes($info->{size}, 1);
 305             my $dirty = render_bytes($info->{dirty}, 1);
 306             return "OK ($dirty of $size dirty)";
 307         }
 308     } elsif ($action eq "invalid") {
 309         return "existing bitmap was invalid and has been cleared";
 310     } else {
 311         return "unknown";
 312     }
 313 };
 314
 315 my $query_backup_status_loop = sub {
 316     my ($self, $vmid, $job_uuid, $qemu_support) = @_;
 317
 318     my $starttime = time ();
 319     my $last_time = $starttime;
 320     my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0);
 321     my ($transferred, $reused);
 322
 323     my $get_mbps = sub {
 324         my ($mb, $delta) = @_;
 325         return "0 B/s" if $mb <= 0;
 326         my $bw = int(($mb / $delta));
 327         return render_bytes($bw, 1) . "/s";
 328     };
 329
 330     my $target = 0;
 331     my $last_reused = 0;
 332     my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'};
 333     my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 334     if ($has_query_bitmap) {
 335         my $total = 0;
 336         my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info');
 337         for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) {
 338             if (!$is_template) {
 339                 my $text = $bitmap_action_to_human->($self, $info);
 340                 my $drive = $info->{drive};
 341                 $drive =~ s/^drive-//; # for consistency
 342                 $self->loginfo("$drive: dirty-bitmap status: $text");
 343             }
 344             $target += $info->{dirty};
 345             $total += $info->{size};
 346             $last_reused += $info->{size} - $info->{dirty};
 347         }
 348         if ($target < $total) {
 349             my $total_h = render_bytes($total, 1);
 350             my $target_h = render_bytes($target, 1);
 351             $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
 352         }
 353     }
 354
 355     my $last_finishing = 0;
 356     while(1) {
 357         my $status = mon_cmd($vmid, 'query-backup');
 358
 359         my $total = $status->{total} || 0;
 360         my $dirty = $status->{dirty};
 361         $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap;
 362         $transferred = $status->{transferred} || 0;
 363         $reused = $status->{reused};
 364         my $percent = $target ? int(($transferred * 100)/$target) : 100;
 365         my $zero = $status->{'zero-bytes'} || 0;
 366
 367         die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $job_uuid);
 368
 369         my $ctime = time();
 370         my $duration = $ctime - $starttime;
 371
 372         my $rbytes = $transferred - $last_transferred;
 373         my $wbytes;
 374         if ($reused) {
 375             # reused includes zero bytes for PBS
 376             $wbytes = $rbytes - ($reused - $last_reused);
 377         } else {
 378             $wbytes = $rbytes - ($zero - $last_zero);
 379         }
 380
 381         my $timediff = ($ctime - $last_time) || 1; # fixme
 382         my $mbps_read = $get_mbps->($rbytes, $timediff);
 383         my $mbps_write = $get_mbps->($wbytes, $timediff);
 384         my $target_h = render_bytes($target, 1);
 385         my $transferred_h = render_bytes($transferred, 1);
 386
 387         my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s"
 388             .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration));
 389
 390         my $res = $status->{status} || 'unknown';
 391         if ($res ne 'active') {
 392             if ($last_percent < 100) {
 393                 $self->loginfo($statusline);
 394             }
 395             if ($res ne 'done') {
 396                 die (($status->{errmsg} || "unknown error") . "\n") if $res eq 'error';
 397                 die "got unexpected status '$res'\n";
 398             }
 399             $last_target = $target if $target;
 400             $last_total = $total if $total;
 401             $last_zero = $zero if $zero;
 402             $last_transferred = $transferred if $transferred;
 403             last;
 404         }
 405         if ($percent != $last_percent && ($timediff > 2)) {
 406             $self->loginfo($statusline);
 407             $last_percent = $percent;
 408             $last_target = $target if $target;
 409             $last_total = $total if $total;
 410             $last_zero = $zero if $zero;
 411             $last_transferred = $transferred if $transferred;
 412             $last_time = $ctime;
 413             $last_reused = $reused;
 414
 415             if (!$last_finishing && $status->{finishing}) {
 416                 $self->loginfo("Waiting for server to finish backup validation...");
 417             }
 418             $last_finishing = $status->{finishing};
 419         }
 420         sleep(1);
 421     }
 422
 423     my $duration = time() - $starttime;
 424
 425     if ($last_zero) {
 426         my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0;
 427         my $zero_h = render_bytes($last_zero);
 428         $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
 429     }
 430     if ($reused) {
 431         my $reused_h = render_bytes($reused);
 432         my $reuse_per = int($reused * 100 / $last_total);
 433         $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
 434     }
 435     if ($transferred) {
 436         my $transferred_h = render_bytes($transferred);
 437         if ($duration) {
 438             my $mbps = $get_mbps->($transferred, $duration);
 439             $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
 440         } else {
 441             $self->loginfo("transferred $transferred_h in <1 seconds");
 442         }
 443     }
 444
 445     return {
 446         total => $last_total,
 447         reused => $reused,
 448     };
 449 };
 450
 451 my $attach_tpmstate_drive = sub {
 452     my ($self, $task, $vmid) = @_;
 453
 454     return if !$task->{tpmpath};
 455
 456     # unconditionally try to remove the tpmstate-named drive - it only exists
 457     # for backing up, and avoids errors if left over from some previous event
 458     eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
 459
 460     $self->loginfo('attaching TPM drive to QEMU for backup');
 461
 462     my $drive = "file=$task->{tpmpath},if=none,read-only=on,id=drive-tpmstate0-backup";
 463     $drive =~ s/\\/\\\\/g;
 464     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
 465     die "attaching TPM drive failed - $ret\n" if $ret !~ m/OK/s;
 466 };
 467
 468 my $detach_tpmstate_drive = sub {
 469     my ($task, $vmid) = @_;
 470     return if !$task->{tpmpath} || !PVE::QemuServer::check_running($vmid);
 471     eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
 472 };
 473
 474 my sub add_backup_performance_options {
 475     my ($qmp_param, $perf, $qemu_support) = @_;
 476
 477     return if !$perf || scalar(keys $perf->%*) == 0;
 478
 479     if (!$qemu_support) {
 480         my $settings_string = join(', ', sort keys $perf->%*);
 481         log_warn("ignoring setting(s): $settings_string - issue checking if supported");
 482         return;
 483     }
 484
 485     if (defined($perf->{'max-workers'})) {
 486         if ($qemu_support->{'backup-max-workers'}) {
 487             $qmp_param->{'max-workers'} = int($perf->{'max-workers'});
 488         } else {
 489             log_warn("ignoring 'max-workers' setting - not supported by running QEMU");
 490         }
 491     }
 492 }
 493
 494 sub get_and_check_pbs_encryption_config {
 495     my ($self) = @_;
 496
 497     my $opts = $self->{vzdump}->{opts};
 498     my $scfg = $opts->{scfg};
 499
 500     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage});
 501     my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage});
 502
 503     if (-e $keyfile) {
 504         if (-e $master_keyfile) {
 505             $self->loginfo("enabling encryption with master key feature");
 506             return ($keyfile, $master_keyfile);
 507         } elsif ($scfg->{'master-pubkey'}) {
 508             die "master public key configured but no key file found\n";
 509         } else {
 510             $self->loginfo("enabling encryption");
 511             return ($keyfile, undef);
 512         }
 513     } else {
 514         my $encryption_fp = $scfg->{'encryption-key'};
 515         die "encryption configured ('$encryption_fp') but no encryption key file found!\n"
 516             if $encryption_fp;
 517         if (-e $master_keyfile) {
 518             $self->log(
 519                 'warn',
 520                 "backup target storage is configured with master-key, but no encryption key set!"
 521                 ." Ignoring master key settings and creating unencrypted backup."
 522             );
 523         }
 524         return (undef, undef);
 525     }
 526     die "internal error - unhandled case for getting & checking PBS encryption ($keyfile, $master_keyfile)!";
 527 }
 528
 529 my sub cleanup_fleecing_images {
 530     my ($self, $disks) = @_;
 531
 532     for my $di ($disks->@*) {
 533         if (my $volid = $di->{'fleece-volid'}) {
 534             eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
 535             $self->log('warn', "error removing fleecing image '$volid' - $@") if $@;
 536         }
 537     }
 538 }
 539
 540 my sub allocate_fleecing_images {
 541     my ($self, $disks, $vmid, $fleecing_storeid, $format) = @_;
 542
 543     die "internal error - no fleecing storage specified\n" if !$fleecing_storeid;
 544
 545     # TODO what about potential left-over images from a failed attempt? Just
 546     # auto-remove? While unlikely, could conflict with manually created image from user...
 547
 548     eval {
 549         my $n = 0; # counter for fleecing image names
 550
 551         for my $di ($disks->@*) {
 552             next if $di->{virtdev} =~ m/^(?:tpmstate|efidisk)\d$/; # too small to be worth it
 553             if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 554                 my $scfg = PVE::Storage::storage_config($self->{storecfg}, $fleecing_storeid);
 555                 my $name = "vm-$vmid-fleece-$n";
 556                 $name .= ".$format" if $scfg->{path};
 557
 558                 my $size = PVE::Tools::convert_size($di->{size}, 'b' => 'kb');
 559
 560                 $di->{'fleece-volid'} = PVE::Storage::vdisk_alloc(
 561                     $self->{storecfg}, $fleecing_storeid, $vmid, $format, $name, $size);
 562
 563                 $n++;
 564             } else {
 565                 die "implement me (type '$di->{type}')";
 566             }
 567         }
 568     };
 569     if (my $err = $@) {
 570         cleanup_fleecing_images($self, $disks);
 571         die $err;
 572     }
 573 }
 574
 575 my sub detach_fleecing_images {
 576     my ($disks, $vmid) = @_;
 577
 578     return if !PVE::QemuServer::Helpers::vm_running_locally($vmid);
 579
 580     for my $di ($disks->@*) {
 581         if (my $volid = $di->{'fleece-volid'}) {
 582             my $devid = "$di->{qmdevice}-fleecing";
 583             $devid =~ s/^drive-//; # re-added by qemu_drivedel()
 584             eval { PVE::QemuServer::qemu_drivedel($vmid, $devid) };
 585         }
 586     }
 587 }
 588
 589 my sub attach_fleecing_images {
 590     my ($self, $disks, $vmid, $format) = @_;
 591
 592     # unconditionally try to remove potential left-overs from a previous backup
 593     detach_fleecing_images($disks, $vmid);
 594
 595     my $vollist = [ map { $_->{'fleece-volid'} } grep { $_->{'fleece-volid'} } $disks->@* ];
 596     PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
 597
 598     for my $di ($disks->@*) {
 599         if (my $volid = $di->{'fleece-volid'}) {
 600             $self->loginfo("$di->{qmdevice}: attaching fleecing image $volid to QEMU");
 601
 602             my $path = PVE::Storage::path($self->{storecfg}, $volid);
 603             my $devid = "$di->{qmdevice}-fleecing";
 604             my $drive = "file=$path,if=none,id=$devid,format=$format,discard=unmap";
 605             # Specify size explicitly, to make it work if storage backend rounded up size for
 606             # fleecing image when allocating.
 607             $drive .= ",size=$di->{size}" if $format eq 'raw';
 608             $drive =~ s/\\/\\\\/g;
 609             my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
 610             die "attaching fleecing image $volid failed - $ret\n" if $ret !~ m/OK/s;
 611         }
 612     }
 613 }
 614
 615 my sub check_and_prepare_fleecing {
 616     my ($self, $vmid, $fleecing_opts, $disks, $is_template, $qemu_support) = @_;
 617
 618     # Even if the VM was started specifically for fleecing, it's possible that the VM is resumed and
 619     # then starts doing IO. For VMs that are not resumed the fleecing images will just stay empty,
 620     # so there is no big cost.
 621
 622     my $use_fleecing = $fleecing_opts && $fleecing_opts->{enabled} && !$is_template;
 623
 624     if ($use_fleecing && !defined($qemu_support->{'backup-fleecing'})) {
 625         $self->log(
 626             'warn',
 627             "running QEMU version does not support backup fleecing - continuing without",
 628         );
 629         $use_fleecing = 0;
 630     }
 631
 632     if ($use_fleecing) {
 633         my ($default_format, $valid_formats) = PVE::Storage::storage_default_format(
 634             $self->{storecfg}, $fleecing_opts->{storage});
 635         my $format = scalar(grep { $_ eq 'qcow2' } $valid_formats->@*) ? 'qcow2' : 'raw';
 636
 637         allocate_fleecing_images($self, $disks, $vmid, $fleecing_opts->{storage}, $format);
 638         attach_fleecing_images($self, $disks, $vmid, $format);
 639     }
 640
 641     return $use_fleecing;
 642 }
 643
 644 sub archive_pbs {
 645     my ($self, $task, $vmid) = @_;
 646
 647     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 648     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 649
 650     my $opts = $self->{vzdump}->{opts};
 651     my $scfg = $opts->{scfg};
 652
 653     my $starttime = time();
 654
 655     my $fingerprint = $scfg->{fingerprint};
 656     my $repo = PVE::PBSClient::get_repository($scfg);
 657     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage});
 658     my ($keyfile, $master_keyfile) = $self->get_and_check_pbs_encryption_config();
 659
 660     my $diskcount = scalar(@{$task->{disks}});
 661     # proxmox-backup-client can only handle raw files and block devs, so only use it (directly) for
 662     # disk-less VMs
 663     if (!$diskcount) {
 664         $self->loginfo("backup contains no disks");
 665
 666         local $ENV{PBS_PASSWORD} = $password;
 667         local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
 668         my $cmd = [
 669             '/usr/bin/proxmox-backup-client',
 670             'backup',
 671             '--repository', $repo,
 672             '--backup-type', 'vm',
 673             '--backup-id', "$vmid",
 674             '--backup-time', $task->{backup_time},
 675         ];
 676         if (defined(my $ns = $scfg->{namespace})) {
 677             push @$cmd, '--ns', $ns;
 678         }
 679         if (defined($keyfile)) {
 680             push @$cmd, '--keyfile', $keyfile;
 681             push @$cmd, '--master-pubkey-file', $master_keyfile if defined($master_keyfile);
 682         }
 683
 684         push @$cmd, "qemu-server.conf:$conffile";
 685         push @$cmd, "fw.conf:$firewall" if -e $firewall;
 686
 687         $self->loginfo("starting template backup");
 688         $self->loginfo(join(' ', @$cmd));
 689
 690         $self->cmd($cmd);
 691
 692         return;
 693     }
 694
 695     # get list early so we die on unkown drive types before doing anything
 696     my $devlist = _get_task_devlist($task);
 697     my $use_fleecing;
 698
 699     $self->enforce_vm_running_for_backup($vmid);
 700     $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
 701
 702     my $backup_job_uuid;
 703     eval {
 704         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 705             die "interrupted by signal\n";
 706         };
 707
 708         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
 709         my $err = $@;
 710         if (!$qemu_support || $err) {
 711             die "query-proxmox-support returned empty value\n" if !$err;
 712             if ($err =~ m/The command query-proxmox-support has not been found/) {
 713                 die "PBS backups are not supported by the running QEMU version. Please make "
 714                   . "sure you've installed the latest version and the VM has been restarted.\n";
 715             } else {
 716                 die "QMP command query-proxmox-support failed - $err\n";
 717             }
 718         }
 719
 720         # pve-qemu supports it since 5.2.0-1 (PVE 6.4), so safe to die since PVE 8
 721         die "master key configured but running QEMU version does not support master keys\n"
 722             if !defined($qemu_support->{'pbs-masterkey'}) && defined($master_keyfile);
 723
 724         $attach_tpmstate_drive->($self, $task, $vmid);
 725
 726         my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 727
 728         $use_fleecing = check_and_prepare_fleecing(
 729             $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support);
 730
 731         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 732
 733         my $params = {
 734             format => "pbs",
 735             'backup-file' => $repo,
 736             'backup-id' => "$vmid",
 737             'backup-time' => $task->{backup_time},
 738             password => $password,
 739             devlist => $devlist,
 740             'config-file' => $conffile,
 741         };
 742         $params->{fleecing} = JSON::true if $use_fleecing;
 743
 744         if (defined(my $ns = $scfg->{namespace})) {
 745             $params->{'backup-ns'} = $ns;
 746         }
 747
 748         $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit};
 749         add_backup_performance_options($params, $opts->{performance}, $qemu_support);
 750
 751         $params->{fingerprint} = $fingerprint if defined($fingerprint);
 752         $params->{'firewall-file'} = $firewall if -e $firewall;
 753
 754         $params->{encrypt} = defined($keyfile) ? JSON::true : JSON::false;
 755         if (defined($keyfile)) {
 756             $params->{keyfile} = $keyfile;
 757             $params->{"master-keyfile"} = $master_keyfile if defined($master_keyfile);
 758         }
 759
 760         $params->{'use-dirty-bitmap'} = JSON::true
 761             if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template;
 762
 763         $params->{timeout} = 125; # give some time to connect to the backup server
 764
 765         my $res = eval { mon_cmd($vmid, "backup", %$params) };
 766         my $qmperr = $@;
 767         $backup_job_uuid = $res->{UUID} if $res;
 768
 769         if ($fs_frozen) {
 770             $self->qga_fs_thaw($vmid);
 771         }
 772
 773         die $qmperr if $qmperr;
 774         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 775
 776         $self->loginfo("started backup task '$backup_job_uuid'");
 777
 778         $self->resume_vm_after_job_start($task, $vmid);
 779
 780         my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support);
 781         $task->{size} = $stat->{total};
 782     };
 783     my $err = $@;
 784     if ($err) {
 785         $self->logerr($err);
 786         $self->mon_backup_cancel($vmid);
 787         $self->resume_vm_after_job_start($task, $vmid);
 788     }
 789     $self->restore_vm_power_state($vmid);
 790
 791     if ($use_fleecing) {
 792         detach_fleecing_images($task->{disks}, $vmid);
 793         cleanup_fleecing_images($self, $task->{disks});
 794     }
 795
 796     die $err if $err;
 797 }
 798
 799 my $fork_compressor_pipe = sub {
 800     my ($self, $comp, $outfileno) = @_;
 801
 802     my @pipefd = POSIX::pipe();
 803     my $cpid = fork();
 804     die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0;
 805     if ($cpid == 0) {
 806         eval {
 807             POSIX::close($pipefd[1]);
 808             # redirect STDIN
 809             my $fd = fileno(STDIN);
 810             close STDIN;
 811             POSIX::close(0) if $fd != 0;
 812             die "unable to redirect STDIN - $!"
 813                 if !open(STDIN, "<&", $pipefd[0]);
 814
 815             # redirect STDOUT
 816             $fd = fileno(STDOUT);
 817             close STDOUT;
 818             POSIX::close (1) if $fd != 1;
 819
 820             die "unable to redirect STDOUT - $!"
 821                 if !open(STDOUT, ">&", $outfileno);
 822
 823             exec($comp);
 824             die "fork compressor '$comp' failed\n";
 825         };
 826         if (my $err = $@) {
 827             $self->logerr($err);
 828             POSIX::_exit(1);
 829         }
 830         POSIX::_exit(0);
 831         kill(-9, $$);
 832     } else {
 833         POSIX::close($pipefd[0]);
 834         $outfileno = $pipefd[1];
 835     }
 836
 837     return ($cpid, $outfileno);
 838 };
 839
 840 sub archive_vma {
 841     my ($self, $task, $vmid, $filename, $comp) = @_;
 842
 843     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 844     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 845
 846     my $opts = $self->{vzdump}->{opts};
 847
 848     my $starttime = time();
 849
 850     my $speed = 0;
 851     if ($opts->{bwlimit}) {
 852         $speed = $opts->{bwlimit}*1024;
 853     }
 854
 855     my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 856
 857     my $diskcount = scalar(@{$task->{disks}});
 858     if ($is_template || !$diskcount) {
 859         my @pathlist;
 860         foreach my $di (@{$task->{disks}}) {
 861             if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 862                 push @pathlist, "$di->{qmdevice}=$di->{path}";
 863             } else {
 864                 die "implement me";
 865             }
 866         }
 867
 868         if (!$diskcount) {
 869             $self->loginfo("backup contains no disks");
 870         }
 871
 872         my $outcmd;
 873         if ($comp) {
 874             $outcmd = "exec:$comp";
 875         } else {
 876             $outcmd = "exec:cat";
 877         }
 878
 879         $outcmd .= " > $filename" if !$opts->{stdout};
 880
 881         my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
 882         push @$cmd, '-c', $firewall if -e $firewall;
 883         push @$cmd, $outcmd, @pathlist;
 884
 885         $self->loginfo("starting template backup");
 886         $self->loginfo(join(' ', @$cmd));
 887
 888         if ($opts->{stdout}) {
 889             $self->cmd($cmd, output => ">&" . fileno($opts->{stdout}));
 890         } else {
 891             $self->cmd($cmd);
 892         }
 893
 894         return;
 895     }
 896
 897     my $devlist = _get_task_devlist($task);
 898     my $use_fleecing;
 899
 900     $self->enforce_vm_running_for_backup($vmid);
 901     $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
 902
 903     my $cpid;
 904     my $backup_job_uuid;
 905
 906     eval {
 907         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 908             die "interrupted by signal\n";
 909         };
 910
 911         # Currently, failing to determine Proxmox support is not critical here, because it's only
 912         # used for performance settings like 'max-workers'.
 913         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
 914         log_warn($@) if $@;
 915
 916         $attach_tpmstate_drive->($self, $task, $vmid);
 917
 918         $use_fleecing = check_and_prepare_fleecing(
 919             $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support);
 920
 921         my $outfh;
 922         if ($opts->{stdout}) {
 923             $outfh = $opts->{stdout};
 924         } else {
 925             $outfh = IO::File->new($filename, "w") ||
 926                 die "unable to open file '$filename' - $!\n";
 927         }
 928         my $outfileno = fileno($outfh);
 929
 930         if ($comp) {
 931             ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno);
 932         }
 933
 934         my $qmpclient = PVE::QMPClient->new();
 935         my $backup_cb = sub {
 936             my ($vmid, $resp) = @_;
 937             $backup_job_uuid = $resp->{return}->{UUID};
 938         };
 939         my $add_fd_cb = sub {
 940             my ($vmid, $resp) = @_;
 941
 942             my $params = {
 943                 'backup-file' => "/dev/fdname/backup",
 944                 speed => $speed,
 945                 'config-file' => $conffile,
 946                 devlist => $devlist
 947             };
 948             $params->{'firewall-file'} = $firewall if -e $firewall;
 949             $params->{fleecing} = JSON::true if $use_fleecing;
 950             add_backup_performance_options($params, $opts->{performance}, $qemu_support);
 951
 952             $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
 953         };
 954
 955         $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd', fd => $outfileno, fdname => "backup");
 956
 957         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 958
 959         eval { $qmpclient->queue_execute(30) };
 960         my $qmperr = $@;
 961
 962         if ($fs_frozen) {
 963             $self->qga_fs_thaw($vmid);
 964         }
 965
 966         die $qmperr if $qmperr;
 967         die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid};
 968
 969         if ($cpid) {
 970             POSIX::close($outfileno) == 0 ||
 971                 die "close output file handle failed\n";
 972         }
 973
 974         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 975
 976         $self->loginfo("started backup task '$backup_job_uuid'");
 977
 978         $self->resume_vm_after_job_start($task, $vmid);
 979
 980         $query_backup_status_loop->($self, $vmid, $backup_job_uuid);
 981     };
 982     my $err = $@;
 983     if ($err) {
 984         $self->logerr($err);
 985         $self->mon_backup_cancel($vmid);
 986         $self->resume_vm_after_job_start($task, $vmid);
 987     }
 988
 989     $self->restore_vm_power_state($vmid);
 990
 991     if ($use_fleecing) {
 992         detach_fleecing_images($task->{disks}, $vmid);
 993         cleanup_fleecing_images($self, $task->{disks});
 994     }
 995
 996     if ($err) {
 997         if ($cpid) {
 998             kill(9, $cpid);
 999             waitpid($cpid, 0);
1000         }
1001         die $err;
1002     }
1003
1004     if ($cpid && (waitpid($cpid, 0) > 0)) {
1005         my $stat = $?;
1006         my $ec = $stat >> 8;
1007         my $signal = $stat & 127;
1008         if ($ec || $signal) {
1009             die "$comp failed - wrong exit status $ec" .
1010                 ($signal ? " (signal $signal)\n" : "\n");
1011         }
1012     }
1013 }
1014
1015 sub _get_task_devlist {
1016     my ($task) = @_;
1017
1018     my $devlist = '';
1019     foreach my $di (@{$task->{disks}}) {
1020         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
1021             $devlist .= ',' if $devlist;
1022             $devlist .= $di->{qmdevice};
1023         } else {
1024             die "implement me (type '$di->{type}')";
1025         }
1026     }
1027     return $devlist;
1028 }
1029
1030 sub qga_fs_freeze {
1031     my ($self, $task, $vmid) = @_;
1032     return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused};
1033
1034     if (!PVE::QemuServer::qga_check_running($vmid, 1)) {
1035         $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?");
1036         return;
1037     }
1038
1039     my $freeze = PVE::QemuServer::get_qga_key($self->{vmlist}->{$vmid}, 'freeze-fs-on-backup') // 1;
1040     if (!$freeze) {
1041         $self->loginfo("skipping guest-agent 'fs-freeze', disabled in VM options");
1042         return;
1043     }
1044
1045     $self->loginfo("issuing guest-agent 'fs-freeze' command");
1046     eval { mon_cmd($vmid, "guest-fsfreeze-freeze") };
1047     $self->logerr($@) if $@;
1048
1049     return 1; # even on mon command error, ensure we always thaw again
1050 }
1051
1052 # only call if fs_freeze return 1
1053 sub qga_fs_thaw {
1054     my ($self, $vmid) = @_;
1055
1056     $self->loginfo("issuing guest-agent 'fs-thaw' command");
1057     eval { mon_cmd($vmid, "guest-fsfreeze-thaw") };
1058     $self->logerr($@) if $@;
1059 }
1060
1061 # we need a running QEMU/KVM process for backup, starts a paused (prelaunch)
1062 # one if VM isn't already running
1063 sub enforce_vm_running_for_backup {
1064     my ($self, $vmid) = @_;
1065
1066     if (PVE::QemuServer::check_running($vmid)) {
1067         $self->{vm_was_running} = 1;
1068         return;
1069     }
1070
1071     eval {
1072         $self->loginfo("starting kvm to execute backup task");
1073         # start with skiplock
1074         my $params = {
1075             skiplock => 1,
1076             skiptemplate => 1,
1077             paused => 1,
1078         };
1079         PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params);
1080     };
1081     die $@ if $@;
1082 }
1083
1084 # resume VM again once in a clear state (stop mode backup of running VM)
1085 sub resume_vm_after_job_start {
1086     my ($self, $task, $vmid) = @_;
1087
1088     return if !$self->{vm_was_running} || $self->{vm_was_paused};
1089
1090     if (my $stoptime = $task->{vmstoptime}) {
1091         my $delay = time() - $task->{vmstoptime};
1092         $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice
1093         $self->loginfo("resuming VM again after $delay seconds");
1094     } else {
1095         $self->loginfo("resuming VM again");
1096     }
1097     mon_cmd($vmid, 'cont', timeout => 45);
1098 }
1099
1100 # stop again if VM was not running before
1101 sub restore_vm_power_state {
1102     my ($self, $vmid) = @_;
1103
1104     # we always let VMs keep running
1105     return if $self->{vm_was_running};
1106
1107     eval {
1108         my $resp = mon_cmd($vmid, 'query-status');
1109         my $status = $resp && $resp->{status} ?  $resp->{status} : 'unknown';
1110         if ($status eq 'prelaunch') {
1111             $self->loginfo("stopping kvm after backup task");
1112             PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1);
1113         } else {
1114             $self->loginfo("kvm status changed after backup ('$status') - keep VM running");
1115         }
1116     };
1117     warn $@ if $@;
1118 }
1119
1120 sub mon_backup_cancel {
1121     my ($self, $vmid) = @_;
1122
1123     $self->loginfo("aborting backup job");
1124     eval { mon_cmd($vmid, 'backup-cancel') };
1125     $self->logerr($@) if $@;
1126 }
1127
1128 sub snapshot {
1129     my ($self, $task, $vmid) = @_;
1130
1131     # nothing to do
1132 }
1133
1134 sub cleanup {
1135     my ($self, $task, $vmid) = @_;
1136
1137     $detach_tpmstate_drive->($task, $vmid);
1138
1139     if ($self->{qmeventd_fh}) {
1140         close($self->{qmeventd_fh});
1141     }
1142 }
1143
1144 1;