PVE/VZDump/QemuServer.pm

   1 package PVE::VZDump::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use IO::File;
   9 use IPC::Open3;
  10 use JSON;
  11 use POSIX qw(EINTR EAGAIN);
  12
  13 use PVE::Cluster qw(cfs_read_file);
  14 use PVE::INotify;
  15 use PVE::IPCC;
  16 use PVE::JSONSchema;
  17 use PVE::PBSClient;
  18 use PVE::QMPClient;
  19 use PVE::Storage::Plugin;
  20 use PVE::Storage::PBSPlugin;
  21 use PVE::Storage;
  22 use PVE::Tools;
  23 use PVE::VZDump;
  24 use PVE::Format qw(render_duration render_bytes);
  25
  26 use PVE::QemuConfig;
  27 use PVE::QemuServer;
  28 use PVE::QemuServer::Machine;
  29 use PVE::QemuServer::Monitor qw(mon_cmd);
  30
  31 use base qw (PVE::VZDump::Plugin);
  32
  33 sub new {
  34     my ($class, $vzdump) = @_;
  35
  36     PVE::VZDump::check_bin('qm');
  37
  38     my $self = bless { vzdump => $vzdump }, $class;
  39
  40     $self->{vmlist} = PVE::QemuServer::vzlist();
  41     $self->{storecfg} = PVE::Storage::config();
  42
  43     return $self;
  44 };
  45
  46 sub type {
  47     return 'qemu';
  48 }
  49
  50 sub vmlist {
  51     my ($self) = @_;
  52     return [ keys %{$self->{vmlist}} ];
  53 }
  54
  55 sub prepare {
  56     my ($self, $task, $vmid, $mode) = @_;
  57
  58     $task->{disks} = [];
  59
  60     my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid);
  61
  62     $self->loginfo("VM Name: $conf->{name}")
  63         if defined($conf->{name});
  64
  65     $self->{vm_was_running} = 1;
  66     $self->{vm_was_paused} = 0;
  67     if (!PVE::QemuServer::check_running($vmid)) {
  68         $self->{vm_was_running} = 0;
  69     } elsif (PVE::QemuServer::vm_is_paused($vmid)) {
  70         $self->{vm_was_paused} = 1;
  71     }
  72
  73     $task->{hostname} = $conf->{name};
  74
  75     my $hostname = PVE::INotify::nodename();
  76
  77     my $vollist = [];
  78     my $drivehash = {};
  79     my $backup_volumes = PVE::QemuConfig->get_backup_volumes($conf);
  80
  81     foreach my $volume (@{$backup_volumes}) {
  82         my $name = $volume->{key};
  83         my $volume_config = $volume->{volume_config};
  84         my $volid = $volume_config->{file};
  85
  86         if (!$volume->{included}) {
  87             $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})");
  88             next;
  89         } elsif ($self->{vm_was_running} && $volume_config->{iothread} &&
  90                  !PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
  91             die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " .
  92                 "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
  93         } else {
  94             my $log = "include disk '$name' '$volid'";
  95             if (defined(my $size = $volume_config->{size})) {
  96                 my $readable_size = PVE::JSONSchema::format_size($size);
  97                 $log .= " $readable_size";
  98             }
  99             $self->loginfo($log);
 100         }
 101
 102         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 103         push @$vollist, $volid if $storeid;
 104         $drivehash->{$name} = $volume->{volume_config};
 105     }
 106
 107     PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
 108
 109     foreach my $ds (sort keys %$drivehash) {
 110         my $drive = $drivehash->{$ds};
 111
 112         my $volid = $drive->{file};
 113         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 114
 115         my $path = $volid;
 116         if ($storeid) {
 117             $path = PVE::Storage::path($self->{storecfg}, $volid);
 118         }
 119         next if !$path;
 120
 121         my ($size, $format) = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) };
 122         die "no such volume '$volid'\n" if $@;
 123
 124         my $diskinfo = {
 125             path => $path,
 126             volid => $volid,
 127             storeid => $storeid,
 128             format => $format,
 129             virtdev => $ds,
 130             qmdevice => "drive-$ds",
 131         };
 132
 133         if ($ds eq 'tpmstate0') {
 134             # TPM drive only exists for backup, which is reflected in the name
 135             $diskinfo->{qmdevice} = 'drive-tpmstate0-backup';
 136             $task->{tpmpath} = $path;
 137         }
 138
 139         if (-b $path) {
 140             $diskinfo->{type} = 'block';
 141         } else {
 142             $diskinfo->{type} = 'file';
 143         }
 144
 145         push @{$task->{disks}}, $diskinfo;
 146     }
 147 }
 148
 149 sub vm_status {
 150     my ($self, $vmid) = @_;
 151
 152     my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0;
 153
 154     return wantarray ? ($running, $running ? 'running' : 'stopped') : $running;
 155 }
 156
 157 sub lock_vm {
 158     my ($self, $vmid) = @_;
 159
 160     PVE::QemuConfig->set_lock($vmid, 'backup');
 161 }
 162
 163 sub unlock_vm {
 164     my ($self, $vmid) = @_;
 165
 166     PVE::QemuConfig->remove_lock($vmid, 'backup');
 167 }
 168
 169 sub stop_vm {
 170     my ($self, $task, $vmid) = @_;
 171
 172     my $opts = $self->{vzdump}->{opts};
 173
 174     my $wait = $opts->{stopwait} * 60;
 175     # send shutdown and wait
 176     $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
 177 }
 178
 179 sub start_vm {
 180     my ($self, $task, $vmid) = @_;
 181
 182     $self->cmd ("qm start $vmid --skiplock");
 183 }
 184
 185 sub suspend_vm {
 186     my ($self, $task, $vmid) = @_;
 187
 188     return if $self->{vm_was_paused};
 189
 190     $self->cmd ("qm suspend $vmid --skiplock");
 191 }
 192
 193 sub resume_vm {
 194     my ($self, $task, $vmid) = @_;
 195
 196     return if $self->{vm_was_paused};
 197
 198     $self->cmd ("qm resume $vmid --skiplock");
 199 }
 200
 201 sub assemble {
 202     my ($self, $task, $vmid) = @_;
 203
 204     my $conffile = PVE::QemuConfig->config_file($vmid);
 205
 206     my $outfile = "$task->{tmpdir}/qemu-server.conf";
 207     my $firewall_src = "/etc/pve/firewall/$vmid.fw";
 208     my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
 209
 210     my $outfd = IO::File->new (">$outfile") ||
 211         die "unable to open '$outfile'";
 212     my $conffd = IO::File->new ($conffile, 'r') ||
 213         die "unable open '$conffile'";
 214
 215     my $found_snapshot;
 216     my $found_pending;
 217     while (defined (my $line = <$conffd>)) {
 218         next if $line =~ m/^\#vzdump\#/; # just to be sure
 219         next if $line =~ m/^\#qmdump\#/; # just to be sure
 220         if ($line =~ m/^\[(.*)\]\s*$/) {
 221             if ($1 =~ m/PENDING/i) {
 222                 $found_pending = 1;
 223             } else {
 224                 $found_snapshot = 1;
 225             }
 226         }
 227         next if $found_snapshot || $found_pending; # skip all snapshots and pending changes config data
 228
 229         if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
 230             $self->loginfo("skip unused drive '$1' (not included into backup)");
 231             next;
 232         }
 233         next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
 234
 235         print $outfd $line;
 236     }
 237
 238     foreach my $di (@{$task->{disks}}) {
 239         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 240             my $storeid = $di->{storeid} || '';
 241             my $format = $di->{format} || '';
 242             print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
 243         } else {
 244             die "internal error";
 245         }
 246     }
 247
 248     if ($found_snapshot) {
 249         $self->loginfo("snapshots found (not included into backup)");
 250     }
 251     if ($found_pending) {
 252         $self->loginfo("pending configuration changes found (not included into backup)");
 253     }
 254
 255     PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src;
 256 }
 257
 258 sub archive {
 259     my ($self, $task, $vmid, $filename, $comp) = @_;
 260
 261     my $opts = $self->{vzdump}->{opts};
 262     my $scfg = $opts->{scfg};
 263
 264     if ($self->{vzdump}->{opts}->{pbs}) {
 265         $self->archive_pbs($task, $vmid);
 266     } else {
 267         $self->archive_vma($task, $vmid, $filename, $comp);
 268     }
 269 }
 270
 271 my $bitmap_action_to_human = sub {
 272     my ($self, $info) = @_;
 273
 274     my $action = $info->{action};
 275
 276     if ($action eq "not-used") {
 277         return "disabled (no support)";
 278     } elsif ($action eq "not-used-removed") {
 279         return "disabled (old bitmap cleared)";
 280     } elsif ($action eq "new") {
 281         return "created new";
 282     } elsif ($action eq "used") {
 283         if ($info->{dirty} == 0) {
 284             return "OK (drive clean)";
 285         } else {
 286             my $size = render_bytes($info->{size}, 1);
 287             my $dirty = render_bytes($info->{dirty}, 1);
 288             return "OK ($dirty of $size dirty)";
 289         }
 290     } elsif ($action eq "invalid") {
 291         return "existing bitmap was invalid and has been cleared";
 292     } else {
 293         return "unknown";
 294     }
 295 };
 296
 297 my $query_backup_status_loop = sub {
 298     my ($self, $vmid, $job_uuid, $qemu_support) = @_;
 299
 300     my $starttime = time ();
 301     my $last_time = $starttime;
 302     my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0);
 303     my ($transferred, $reused);
 304
 305     my $get_mbps = sub {
 306         my ($mb, $delta) = @_;
 307         return "0 B/s" if $mb <= 0;
 308         my $bw = int(($mb / $delta));
 309         return render_bytes($bw, 1) . "/s";
 310     };
 311
 312     my $target = 0;
 313     my $last_reused = 0;
 314     my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'};
 315     my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 316     if ($has_query_bitmap) {
 317         my $total = 0;
 318         my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info');
 319         for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) {
 320             if (!$is_template) {
 321                 my $text = $bitmap_action_to_human->($self, $info);
 322                 my $drive = $info->{drive};
 323                 $drive =~ s/^drive-//; # for consistency
 324                 $self->loginfo("$drive: dirty-bitmap status: $text");
 325             }
 326             $target += $info->{dirty};
 327             $total += $info->{size};
 328             $last_reused += $info->{size} - $info->{dirty};
 329         }
 330         if ($target < $total) {
 331             my $total_h = render_bytes($total, 1);
 332             my $target_h = render_bytes($target, 1);
 333             $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
 334         }
 335     }
 336
 337     my $last_finishing = 0;
 338     while(1) {
 339         my $status = mon_cmd($vmid, 'query-backup');
 340
 341         my $total = $status->{total} || 0;
 342         my $dirty = $status->{dirty};
 343         $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap;
 344         $transferred = $status->{transferred} || 0;
 345         $reused = $status->{reused};
 346         my $percent = $target ? int(($transferred * 100)/$target) : 100;
 347         my $zero = $status->{'zero-bytes'} || 0;
 348
 349         die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $job_uuid);
 350
 351         my $ctime = time();
 352         my $duration = $ctime - $starttime;
 353
 354         my $rbytes = $transferred - $last_transferred;
 355         my $wbytes;
 356         if ($reused) {
 357             # reused includes zero bytes for PBS
 358             $wbytes = $rbytes - ($reused - $last_reused);
 359         } else {
 360             $wbytes = $rbytes - ($zero - $last_zero);
 361         }
 362
 363         my $timediff = ($ctime - $last_time) || 1; # fixme
 364         my $mbps_read = $get_mbps->($rbytes, $timediff);
 365         my $mbps_write = $get_mbps->($wbytes, $timediff);
 366         my $target_h = render_bytes($target, 1);
 367         my $transferred_h = render_bytes($transferred, 1);
 368
 369         my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s"
 370             .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration));
 371
 372         my $res = $status->{status} || 'unknown';
 373         if ($res ne 'active') {
 374             if ($last_percent < 100) {
 375                 $self->loginfo($statusline);
 376             }
 377             if ($res ne 'done') {
 378                 die (($status->{errmsg} || "unknown error") . "\n") if $res eq 'error';
 379                 die "got unexpected status '$res'\n";
 380             }
 381             $last_target = $target if $target;
 382             $last_total = $total if $total;
 383             $last_zero = $zero if $zero;
 384             $last_transferred = $transferred if $transferred;
 385             last;
 386         }
 387         if ($percent != $last_percent && ($timediff > 2)) {
 388             $self->loginfo($statusline);
 389             $last_percent = $percent;
 390             $last_target = $target if $target;
 391             $last_total = $total if $total;
 392             $last_zero = $zero if $zero;
 393             $last_transferred = $transferred if $transferred;
 394             $last_time = $ctime;
 395             $last_reused = $reused;
 396
 397             if (!$last_finishing && $status->{finishing}) {
 398                 $self->loginfo("Waiting for server to finish backup validation...");
 399             }
 400             $last_finishing = $status->{finishing};
 401         }
 402         sleep(1);
 403     }
 404
 405     my $duration = time() - $starttime;
 406
 407     if ($last_zero) {
 408         my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0;
 409         my $zero_h = render_bytes($last_zero);
 410         $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
 411     }
 412     if ($reused) {
 413         my $reused_h = render_bytes($reused);
 414         my $reuse_per = int($reused * 100 / $last_total);
 415         $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
 416     }
 417     if ($transferred) {
 418         my $transferred_h = render_bytes($transferred);
 419         if ($duration) {
 420             my $mbps = $get_mbps->($transferred, $duration);
 421             $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
 422         } else {
 423             $self->loginfo("transferred $transferred_h in <1 seconds");
 424         }
 425     }
 426
 427     return {
 428         total => $last_total,
 429         reused => $reused,
 430     };
 431 };
 432
 433 my $attach_tpmstate_drive = sub {
 434     my ($self, $task, $vmid) = @_;
 435
 436     return if !$task->{tpmpath};
 437
 438     # unconditionally try to remove the tpmstate-named drive - it only exists
 439     # for backing up, and avoids errors if left over from some previous event
 440     eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
 441
 442     $self->loginfo('attaching TPM drive to QEMU for backup');
 443
 444     my $drive = "file=$task->{tpmpath},if=none,read-only=on,id=drive-tpmstate0-backup";
 445     $drive =~ s/\\/\\\\/g;
 446     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
 447     die "attaching TPM drive failed - $ret\n" if $ret !~ m/OK/s;
 448 };
 449
 450 my $detach_tpmstate_drive = sub {
 451     my ($task, $vmid) = @_;
 452     return if !$task->{tpmpath} || !PVE::QemuServer::check_running($vmid);
 453     eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
 454 };
 455
 456 sub archive_pbs {
 457     my ($self, $task, $vmid) = @_;
 458
 459     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 460     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 461
 462     my $opts = $self->{vzdump}->{opts};
 463     my $scfg = $opts->{scfg};
 464
 465     my $starttime = time();
 466
 467     my $fingerprint = $scfg->{fingerprint};
 468     my $repo = PVE::PBSClient::get_repository($scfg);
 469     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage});
 470     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage});
 471     my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage});
 472
 473     my $diskcount = scalar(@{$task->{disks}});
 474     # proxmox-backup-client can only handle raw files and block devs
 475     # only use it (directly) for disk-less VMs
 476     if (!$diskcount) {
 477         my @pathlist;
 478         $self->loginfo("backup contains no disks");
 479
 480         local $ENV{PBS_PASSWORD} = $password;
 481         local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
 482         my $cmd = [
 483             '/usr/bin/proxmox-backup-client',
 484             'backup',
 485             '--repository', $repo,
 486             '--backup-type', 'vm',
 487             '--backup-id', "$vmid",
 488             '--backup-time', $task->{backup_time},
 489         ];
 490         if (defined(my $ns = $scfg->{namespace})) {
 491             push @$cmd, '--ns', $ns;
 492         }
 493
 494         push @$cmd, "qemu-server.conf:$conffile";
 495         push @$cmd, "fw.conf:$firewall" if -e $firewall;
 496
 497         $self->loginfo("starting template backup");
 498         $self->loginfo(join(' ', @$cmd));
 499
 500         $self->cmd($cmd);
 501
 502         return;
 503     }
 504
 505     # get list early so we die on unkown drive types before doing anything
 506     my $devlist = _get_task_devlist($task);
 507
 508     $self->enforce_vm_running_for_backup($vmid);
 509     $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
 510
 511     my $backup_job_uuid;
 512     eval {
 513         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 514             die "interrupted by signal\n";
 515         };
 516
 517         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
 518         my $err = $@;
 519         if (!$qemu_support || $err) {
 520             die "query-proxmox-support returned empty value\n" if !$err;
 521             if ($err =~ m/The command query-proxmox-support has not been found/) {
 522                 die "PBS backups are not supported by the running QEMU version. Please make "
 523                   . "sure you've installed the latest version and the VM has been restarted.\n";
 524             } else {
 525                 die "QMP command query-proxmox-support failed - $err\n";
 526             }
 527         }
 528
 529         if (!defined($qemu_support->{"pbs-masterkey"}) && -e $master_keyfile) {
 530             $self->loginfo("WARNING: backup target is configured with master key, but running QEMU version does not support master keys.");
 531             $self->loginfo("Please make sure you've installed the latest version and the VM has been restarted to use master key feature.");
 532             $master_keyfile = undef; # skip rest of master key handling below
 533         }
 534
 535         $attach_tpmstate_drive->($self, $task, $vmid);
 536
 537         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 538
 539         my $params = {
 540             format => "pbs",
 541             'backup-file' => $repo,
 542             'backup-id' => "$vmid",
 543             'backup-time' => $task->{backup_time},
 544             password => $password,
 545             devlist => $devlist,
 546             'config-file' => $conffile,
 547         };
 548         if (defined(my $ns = $scfg->{namespace})) {
 549             $params->{'backup-ns'} = $ns;
 550         }
 551         $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit};
 552         $params->{fingerprint} = $fingerprint if defined($fingerprint);
 553         $params->{'firewall-file'} = $firewall if -e $firewall;
 554         if (-e $keyfile) {
 555             $self->loginfo("enabling encryption");
 556             $params->{keyfile} = $keyfile;
 557             $params->{encrypt} = JSON::true;
 558             if (defined($master_keyfile)) {
 559                 if (-e $master_keyfile) {
 560                     $self->loginfo("enabling master key feature");
 561                     $params->{"master-keyfile"} = $master_keyfile;
 562                 } elsif ($scfg->{'master-pubkey'}) {
 563                     die "master public key configured but no key file found\n";
 564                 }
 565             }
 566         } else {
 567             my $encryption_fp = $scfg->{'encryption-key'};
 568             die "encryption configured ('$encryption_fp') but no encryption key file found!\n"
 569                 if $encryption_fp;
 570             $self->loginfo("WARNING: backup target is configured with master key, but this backup is not encrypted - master key settings will be ignored!")
 571                 if defined($master_keyfile) && -e $master_keyfile;
 572             $params->{encrypt} = JSON::false;
 573         }
 574
 575         my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 576         $params->{'use-dirty-bitmap'} = JSON::true
 577             if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template;
 578
 579         $params->{timeout} = 125; # give some time to connect to the backup server
 580
 581         my $res = eval { mon_cmd($vmid, "backup", %$params) };
 582         my $qmperr = $@;
 583         $backup_job_uuid = $res->{UUID} if $res;
 584
 585         if ($fs_frozen) {
 586             $self->qga_fs_thaw($vmid);
 587         }
 588
 589         die $qmperr if $qmperr;
 590         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 591
 592         $self->loginfo("started backup task '$backup_job_uuid'");
 593
 594         $self->resume_vm_after_job_start($task, $vmid);
 595
 596         my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support);
 597         $task->{size} = $stat->{total};
 598     };
 599     my $err = $@;
 600     if ($err) {
 601         $self->logerr($err);
 602         $self->mon_backup_cancel($vmid);
 603         $self->resume_vm_after_job_start($task, $vmid);
 604     }
 605     $self->restore_vm_power_state($vmid);
 606
 607     die $err if $err;
 608 }
 609
 610 my $fork_compressor_pipe = sub {
 611     my ($self, $comp, $outfileno) = @_;
 612
 613     my @pipefd = POSIX::pipe();
 614     my $cpid = fork();
 615     die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0;
 616     if ($cpid == 0) {
 617         eval {
 618             POSIX::close($pipefd[1]);
 619             # redirect STDIN
 620             my $fd = fileno(STDIN);
 621             close STDIN;
 622             POSIX::close(0) if $fd != 0;
 623             die "unable to redirect STDIN - $!"
 624                 if !open(STDIN, "<&", $pipefd[0]);
 625
 626             # redirect STDOUT
 627             $fd = fileno(STDOUT);
 628             close STDOUT;
 629             POSIX::close (1) if $fd != 1;
 630
 631             die "unable to redirect STDOUT - $!"
 632                 if !open(STDOUT, ">&", $outfileno);
 633
 634             exec($comp);
 635             die "fork compressor '$comp' failed\n";
 636         };
 637         if (my $err = $@) {
 638             $self->logerr($err);
 639             POSIX::_exit(1);
 640         }
 641         POSIX::_exit(0);
 642         kill(-9, $$);
 643     } else {
 644         POSIX::close($pipefd[0]);
 645         $outfileno = $pipefd[1];
 646     }
 647
 648     return ($cpid, $outfileno);
 649 };
 650
 651 sub archive_vma {
 652     my ($self, $task, $vmid, $filename, $comp) = @_;
 653
 654     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 655     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 656
 657     my $opts = $self->{vzdump}->{opts};
 658
 659     my $starttime = time();
 660
 661     my $speed = 0;
 662     if ($opts->{bwlimit}) {
 663         $speed = $opts->{bwlimit}*1024;
 664     }
 665
 666     my $diskcount = scalar(@{$task->{disks}});
 667     if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) {
 668         my @pathlist;
 669         foreach my $di (@{$task->{disks}}) {
 670             if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 671                 push @pathlist, "$di->{qmdevice}=$di->{path}";
 672             } else {
 673                 die "implement me";
 674             }
 675         }
 676
 677         if (!$diskcount) {
 678             $self->loginfo("backup contains no disks");
 679         }
 680
 681         my $outcmd;
 682         if ($comp) {
 683             $outcmd = "exec:$comp";
 684         } else {
 685             $outcmd = "exec:cat";
 686         }
 687
 688         $outcmd .= " > $filename" if !$opts->{stdout};
 689
 690         my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
 691         push @$cmd, '-c', $firewall if -e $firewall;
 692         push @$cmd, $outcmd, @pathlist;
 693
 694         $self->loginfo("starting template backup");
 695         $self->loginfo(join(' ', @$cmd));
 696
 697         if ($opts->{stdout}) {
 698             $self->cmd($cmd, output => ">&" . fileno($opts->{stdout}));
 699         } else {
 700             $self->cmd($cmd);
 701         }
 702
 703         return;
 704     }
 705
 706     my $devlist = _get_task_devlist($task);
 707
 708     $self->enforce_vm_running_for_backup($vmid);
 709     $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
 710
 711     my $cpid;
 712     my $backup_job_uuid;
 713
 714     eval {
 715         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 716             die "interrupted by signal\n";
 717         };
 718
 719         $attach_tpmstate_drive->($self, $task, $vmid);
 720
 721         my $outfh;
 722         if ($opts->{stdout}) {
 723             $outfh = $opts->{stdout};
 724         } else {
 725             $outfh = IO::File->new($filename, "w") ||
 726                 die "unable to open file '$filename' - $!\n";
 727         }
 728         my $outfileno = fileno($outfh);
 729
 730         if ($comp) {
 731             ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno);
 732         }
 733
 734         my $qmpclient = PVE::QMPClient->new();
 735         my $backup_cb = sub {
 736             my ($vmid, $resp) = @_;
 737             $backup_job_uuid = $resp->{return}->{UUID};
 738         };
 739         my $add_fd_cb = sub {
 740             my ($vmid, $resp) = @_;
 741
 742             my $params = {
 743                 'backup-file' => "/dev/fdname/backup",
 744                 speed => $speed,
 745                 'config-file' => $conffile,
 746                 devlist => $devlist
 747             };
 748             $params->{'firewall-file'} = $firewall if -e $firewall;
 749
 750             $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
 751         };
 752
 753         $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd', fd => $outfileno, fdname => "backup");
 754
 755         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 756
 757         eval { $qmpclient->queue_execute(30) };
 758         my $qmperr = $@;
 759
 760         if ($fs_frozen) {
 761             $self->qga_fs_thaw($vmid);
 762         }
 763
 764         die $qmperr if $qmperr;
 765         die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid};
 766
 767         if ($cpid) {
 768             POSIX::close($outfileno) == 0 ||
 769                 die "close output file handle failed\n";
 770         }
 771
 772         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 773
 774         $self->loginfo("started backup task '$backup_job_uuid'");
 775
 776         $self->resume_vm_after_job_start($task, $vmid);
 777
 778         $query_backup_status_loop->($self, $vmid, $backup_job_uuid);
 779     };
 780     my $err = $@;
 781     if ($err) {
 782         $self->logerr($err);
 783         $self->mon_backup_cancel($vmid);
 784         $self->resume_vm_after_job_start($task, $vmid);
 785     }
 786
 787     $self->restore_vm_power_state($vmid);
 788
 789     if ($err) {
 790         if ($cpid) {
 791             kill(9, $cpid);
 792             waitpid($cpid, 0);
 793         }
 794         die $err;
 795     }
 796
 797     if ($cpid && (waitpid($cpid, 0) > 0)) {
 798         my $stat = $?;
 799         my $ec = $stat >> 8;
 800         my $signal = $stat & 127;
 801         if ($ec || $signal) {
 802             die "$comp failed - wrong exit status $ec" .
 803                 ($signal ? " (signal $signal)\n" : "\n");
 804         }
 805     }
 806 }
 807
 808 sub _get_task_devlist {
 809     my ($task) = @_;
 810
 811     my $devlist = '';
 812     foreach my $di (@{$task->{disks}}) {
 813         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 814             $devlist .= ',' if $devlist;
 815             $devlist .= $di->{qmdevice};
 816         } else {
 817             die "implement me (type '$di->{type}')";
 818         }
 819     }
 820     return $devlist;
 821 }
 822
 823 sub qga_fs_freeze {
 824     my ($self, $task, $vmid) = @_;
 825     return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused};
 826
 827     if (!PVE::QemuServer::qga_check_running($vmid, 1)) {
 828         $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?");
 829         return;
 830     }
 831
 832     $self->loginfo("issuing guest-agent 'fs-freeze' command");
 833     eval { mon_cmd($vmid, "guest-fsfreeze-freeze") };
 834     $self->logerr($@) if $@;
 835
 836     return 1; # even on mon command error, ensure we always thaw again
 837 }
 838
 839 # only call if fs_freeze return 1
 840 sub qga_fs_thaw {
 841     my ($self, $vmid) = @_;
 842
 843     $self->loginfo("issuing guest-agent 'fs-thaw' command");
 844     eval { mon_cmd($vmid, "guest-fsfreeze-thaw") };
 845     $self->logerr($@) if $@;
 846 }
 847
 848 # we need a running QEMU/KVM process for backup, starts a paused (prelaunch)
 849 # one if VM isn't already running
 850 sub enforce_vm_running_for_backup {
 851     my ($self, $vmid) = @_;
 852
 853     if (PVE::QemuServer::check_running($vmid)) {
 854         $self->{vm_was_running} = 1;
 855         return;
 856     }
 857
 858     eval {
 859         $self->loginfo("starting kvm to execute backup task");
 860         # start with skiplock
 861         my $params = {
 862             skiplock => 1,
 863             skiptemplate => 1,
 864             paused => 1,
 865         };
 866         PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params);
 867     };
 868     die $@ if $@;
 869 }
 870
 871 # resume VM again once in a clear state (stop mode backup of running VM)
 872 sub resume_vm_after_job_start {
 873     my ($self, $task, $vmid) = @_;
 874
 875     return if !$self->{vm_was_running} || $self->{vm_was_paused};
 876
 877     if (my $stoptime = $task->{vmstoptime}) {
 878         my $delay = time() - $task->{vmstoptime};
 879         $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice
 880         $self->loginfo("resuming VM again after $delay seconds");
 881     } else {
 882         $self->loginfo("resuming VM again");
 883     }
 884     mon_cmd($vmid, 'cont', timeout => 45);
 885 }
 886
 887 # stop again if VM was not running before
 888 sub restore_vm_power_state {
 889     my ($self, $vmid) = @_;
 890
 891     # we always let VMs keep running
 892     return if $self->{vm_was_running};
 893
 894     eval {
 895         my $resp = mon_cmd($vmid, 'query-status');
 896         my $status = $resp && $resp->{status} ?  $resp->{status} : 'unknown';
 897         if ($status eq 'prelaunch') {
 898             $self->loginfo("stopping kvm after backup task");
 899             PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1);
 900         } else {
 901             $self->loginfo("kvm status changed after backup ('$status') - keep VM running");
 902         }
 903     };
 904     warn $@ if $@;
 905 }
 906
 907 sub mon_backup_cancel {
 908     my ($self, $vmid) = @_;
 909
 910     $self->loginfo("aborting backup job");
 911     eval { mon_cmd($vmid, 'backup-cancel') };
 912     $self->logerr($@) if $@;
 913 }
 914
 915 sub snapshot {
 916     my ($self, $task, $vmid) = @_;
 917
 918     # nothing to do
 919 }
 920
 921 sub cleanup {
 922     my ($self, $task, $vmid) = @_;
 923
 924     $detach_tpmstate_drive->($task, $vmid);
 925
 926     if ($self->{qmeventd_fh}) {
 927         close($self->{qmeventd_fh});
 928     }
 929 }
 930
 931 1;